using Newtonsoft.Json; using System.Linq; using System.Diagnostics; using System.Text.RegularExpressions; using ttrss_co_client.tasks; namespace ttrss_co_client { class Program { private static ttrss.ApiClient ttrssClient; static async Task Main(string[] args) { // var tasktypes = AppDomain.CurrentDomain.GetAssemblies() // .SelectMany(domainAssembly => domainAssembly.GetTypes()) // .Where(type => type.IsSubclassOf(typeof(Phase1Task)) && !type.IsAbstract) // .ToList(); // foreach (var subtype in subtypes) // { // var inst = (CoClientTask) Activator.CreateInstance(subtype); // } var conf = Configure(); Console.WriteLine($"{DateTime.Now.ToLongTimeString()}"); ttrssClient = new ttrss.ApiClient(conf.BaseURI); await ttrssClient.Login(conf.Username, conf.Password); var loggedin = await ttrssClient.IsLoggedIn(); Console.WriteLine($"logged in: {loggedin}"); #region phase 1 Console.WriteLine("===== phase 1 ====="); var Phase1Tasks = new List>(); Phase1Task.TtrssClient = ttrssClient; Phase1Task.Conf = conf; var phase1TaskTypes = AppDomain.CurrentDomain.GetAssemblies() .SelectMany(domainAssembly => domainAssembly.GetTypes()) .Where(type => type.IsSubclassOf(typeof(Phase1Task)) && !type.IsAbstract) .ToList(); var phase1TaskConcretions = new List(); foreach (var phase1TaskType in phase1TaskTypes) { var concretion = (Phase1Task) Activator.CreateInstance(phase1TaskType); concretion.TriggerLabel = conf.feedActions.FirstOrDefault(fa => fa.command == concretion.TaskName).triggerlabelCaption; phase1TaskConcretions.Add(concretion); } Console.WriteLine($"{phase1TaskConcretions.Count()} phase 1 task types understood"); var unreadFeeds = await ttrssClient.GetFeeds(cat_id: -3, unread_only: true); Console.WriteLine($"{unreadFeeds.Count()} feeds unread"); foreach (var uf in unreadFeeds) { Console.WriteLine($"unread feed: {uf.title}"); var headlines = await ttrssClient.GetHeadlines(uf.id, view_mode: ttrss.ApiClient.VIEWMODE.Unread, include_attachments: true); foreach (var hl in headlines) { var labelsWRTArticle = (await ttrssClient.GetLabels(hl.id)); var actionsForFeed = conf.feedActions.Where(fa => labelsWRTArticle.Where(l => l.@checked).Select(l => l.caption).Contains(fa.triggerlabelCaption))?.ToList(); if (actionsForFeed != null && actionsForFeed.Any()) { var action = actionsForFeed.First(); Console.WriteLine($" headline {hl.title} has label {action.triggerlabelCaption} / action {action.command}"); var appropriatePhase1Task = phase1TaskConcretions.FirstOrDefault(tt => tt.TaskName == action.command); if(appropriatePhase1Task != null) { Phase1Tasks.Add(appropriatePhase1Task.ActOn(hl, labelsWRTArticle)); } else { Console.Error.WriteLine($"couldn't find phase 1 task {action.command} for workorder referring to article id {hl.id}!"); } } } } Console.WriteLine($"done processing feeds. phase 1 tasks launched. wait to complete."); var remainingWork = new List(); foreach(var lingeringTask in Phase1Tasks) { var wo = await lingeringTask; if(wo != null) { Console.WriteLine($"articleId {wo.articleId} left a work order; it has {wo.Phase2TaskList.Count()} phase 2 task{(wo.Phase2TaskList.Count() == 1 ? "" : "s")}"); remainingWork.Add(wo); } } Console.WriteLine($"phase 1 tasks complete, carrying {remainingWork.Count()} workorder{(remainingWork.Count() == 1 ? "" : "s")} to phase 2"); #endregion #region local tasks (stripping ads, converting.) Console.WriteLine("===== phase 2 ====="); //loop through working directory looking for other work orders to add var workOrderFiles = Directory.GetFiles($"{conf.WorkingDirectory}", "workorder.json", new EnumerationOptions(){RecurseSubdirectories = true}); Console.WriteLine($"{workOrderFiles.Count()} workorder file{(workOrderFiles.Count() == 1 ? "" : "s")} located"); foreach(var workorderFile in workOrderFiles) { try { remainingWork.Add(JsonConvert.DeserializeObject(workorderFile)); Console.WriteLine($"picked up workorder task; {workorderFile}"); } catch(Exception e) { Console.Error.WriteLine($"error picking up work order file {workorderFile} - {e.Message}; {e.StackTrace}"); } } Console.WriteLine($"{remainingWork.Count} phase 2 workorders, between pulled from ttrss and read from files"); Phase2Task.TtrssClient = ttrssClient; Phase2Task.Conf = conf; var phase2TaskTypes = AppDomain.CurrentDomain.GetAssemblies() .SelectMany(domainAssembly => domainAssembly.GetTypes()) .Where(type => type.IsSubclassOf(typeof(Phase2Task)) && !type.IsAbstract) .ToList(); var phase2TaskConcretions = new List(); foreach (var phase2TaskType in phase2TaskTypes) { var concretion = (Phase2Task) Activator.CreateInstance(phase2TaskType); phase2TaskConcretions.Add(concretion); } while(remainingWork.Count > 0) { //todo: solve the halting problem //ok but seriously, maybe I could time out work orders after a while? but converting a 3-hour 4k video could easily take an hour! Console.WriteLine($"{phase2TaskConcretions.Count()} phase 2 task types understood"); var Phase2Tasks = new List>>(); Console.WriteLine($"launching first pass over work orders in phase 2."); foreach(var wo in remainingWork) { var taskName = wo.Phase2TaskList[wo.Phase2TaskList.Keys.Min()]; var appropriatePhase2Task = phase2TaskConcretions.FirstOrDefault(tt => tt.TaskName == taskName); if(appropriatePhase2Task != null) { wo.Phase2TaskList.Remove(wo.Phase2TaskList.Keys.Min()); Phase2Tasks.Add(appropriatePhase2Task.ActOn(wo)); } else { Console.Error.WriteLine($"couldn't find phase 2 task {taskName} for workorder referring to article id {wo.articleId}!"); } } Console.WriteLine($"phase 2 tasks launched. now the complex part."); remainingWork = new List(); foreach(var lingeringTask in Phase2Tasks) { var wo = await lingeringTask; //if you tell me it's done, or you need to continue now.... I believe you. //TODO: be smart enough to override? switch (wo.Item1) { case Phase2Task.TaskStatus.Done: // :) break; case Phase2Task.TaskStatus.ContinueNow: remainingWork.Add(wo.Item2); break; case Phase2Task.TaskStatus.TryLater: File.WriteAllText(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString(), "workorder.json"), JsonConvert.SerializeObject(wo)); break; } } Console.WriteLine($"{remainingWork.Count} phase 2 tasks to be re-looped."); } #endregion await ttrssClient.Logout(); Console.WriteLine($"logged out of ttrss."); Console.WriteLine($"done for real"); } static Configuration Configure(string configurationPath = "appsettings.json") { if (!File.Exists(configurationPath)) { Console.Error.WriteLine($"could not find configuration at {configurationPath}! copying sample to that spot."); File.Copy("sample-appsettings.json", configurationPath); //and you know what, if that explodes at the OS level, the OS should give you an error return null; } var fileContents = File.ReadAllText(configurationPath); if (string.IsNullOrWhiteSpace(fileContents)) { Console.Error.WriteLine($"configuration file at {configurationPath} was empty! overwriting with sample settings."); File.Copy("sample-appsettings.json", configurationPath, true); return null; } var conf = JsonConvert.DeserializeObject(fileContents); if (conf == null) { Console.Error.WriteLine($"configuration file at {configurationPath} was empty! overwriting with sample settings."); File.Copy("sample-appsettings.json", configurationPath, true); return null; } return conf; } private static async Task> podcastifyYT(string articleLink, string podcastName) { Console.WriteLine($" youtube-podcastifying {articleLink} ({podcastName})"); try { var ytdl = new YoutubeDLSharp.YoutubeDL(); ytdl.YoutubeDLPath = "yt-dlp"; ytdl.FFmpegPath = "ffmpeg"; ytdl.OutputFolder = $"./tmp/Podcasts/{podcastName}"; ytdl.OutputFileTemplate = "%(upload_date)s - %(title)s - [%(id)s].%(ext)s"; var sw = new Stopwatch(); sw.Start(); var res = await ytdl.RunAudioDownload(articleLink); sw.Stop(); var outputStr = $"{(res.Success ? "Success" : "fail")} in {sw.Elapsed}"; if (res.ErrorOutput != null && res.ErrorOutput.Length > 0) { outputStr += "\n" + string.Join('\n', res.ErrorOutput); } Console.WriteLine($" {(res.Success ? "Success" : "fail")} in {sw.Elapsed} - {res.Data}"); if (res.Success && !res.Data.EndsWith(".mp3")) { Console.WriteLine(" must convert audio"); sw.Reset(); var outputFilename = res.Data.Substring(0, res.Data.LastIndexOf('.')) + ".mp3"; sw.Start(); var conversionProc = Process.Start("ffmpeg", $"-y -i \"{res.Data}\" \"{outputFilename}\""); conversionProc.WaitForExit(); sw.Stop(); File.Delete(res.Data); outputStr += $"\nconverted in {sw.Elapsed}"; } return new Tuple(res.Success, outputStr); } catch (Exception e) { Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}"); return new Tuple(false, $"{e.ToString()}: {e.Message}.\n{e.StackTrace}"); } } private static async Task> podcastifyAttachment(string attachmentLink, string podcastName, string episodeTitle) { Console.WriteLine($" attachment-podcastifying {attachmentLink} ({podcastName})"); try { var extensionUpstream = attachmentLink.Substring(attachmentLink.LastIndexOf('.')); var containingDirectory = $"./tmp/Podcasts/{podcastName}/"; var outputFilename = $"{containingDirectory}{episodeTitle}{extensionUpstream}"; if (!Directory.Exists(containingDirectory)) { Directory.CreateDirectory(containingDirectory); } var downloader = new HttpClient(); var sw = new Stopwatch(); sw.Start(); File.WriteAllBytes(outputFilename, await (await downloader.GetAsync(attachmentLink)).Content.ReadAsByteArrayAsync()); sw.Stop(); var outputStr = $"{(File.Exists(outputFilename) ? "Success" : "fail")} in {sw.Elapsed}"; Console.WriteLine($" {outputStr} - {outputFilename}"); if (File.Exists(outputFilename) && extensionUpstream != ".mp3") { Console.WriteLine(" must convert audio"); sw.Reset(); var targetFilename = outputFilename.Substring(0, outputFilename.LastIndexOf('.')) + ".mp3"; sw.Start(); var conversionProc = Process.Start("ffmpeg", $"-y -i \"{outputFilename}\" \"{targetFilename}\""); conversionProc.WaitForExit(); sw.Stop(); File.Delete(outputFilename); outputStr += $"\nconverted in {sw.Elapsed}"; } return new Tuple(true, outputStr); } catch (Exception e) { Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}"); return new Tuple(false, $"{e.ToString()}: {e.Message}.\n{e.StackTrace}"); } } // private static async Task> SponsorCheck(ttrss.datastructures.Headline hl) // { // if (!hl.link.Host.EndsWith("youtube.com")) // { // return new Tuple(true, "sponsorblock, sadly, only exists for youtube"); // } // var match = Regex.Match(hl.link.Query, "v=([^&]+)(&|$)"); // var videoId = match.Groups?[1].Value; // var c = new HttpClient(); // var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview"); // if (sponsorblockcheck.StatusCode == System.Net.HttpStatusCode.NotFound) // { // Console.WriteLine($"sponsorblock reports that {videoId} has no entries (yet)"); // var updateTimestamp = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); // updateTimestamp = updateTimestamp.AddSeconds(hl.updated).ToLocalTime(); // if (DateTime.Now - updateTimestamp > TimeSpan.FromMinutes(45)) // { // return new Tuple(true, $"updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock"); // } // else // { // return new Tuple(false, "none found, waiting"); // } // } // else // { // try // { // var segments = JsonConvert.DeserializeObject>(await sponsorblockcheck.Content.ReadAsStringAsync()); // if (segments.Count() > 1) // { // return new Tuple(true, $"{segments.Count()} segments"); // } // else // { // return new Tuple(false, $"no segments"); // } // } // catch (Exception e) // { // return new Tuple(false, $"{e.ToString()} - {e.Message}"); // } // } // } } }