diff --git a/.gitignore b/.gitignore index 2f84bbf..52b4a3e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -tmp/ +working/ # ---> VisualStudio ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. diff --git a/Configuration.cs b/Configuration.cs index a1755d6..13d1e84 100644 --- a/Configuration.cs +++ b/Configuration.cs @@ -9,6 +9,7 @@ namespace ttrss_co_client public string Password { get; set; } public string PodcastTitlePrefix { get; set; } public string OnDoneCopy { get; set; } + public string WorkingDirectory { get; set; } = "./working/"; public IEnumerable feedActions { get; set; } public class FeedAction { diff --git a/Program.cs b/Program.cs index 55211b2..71172e7 100644 --- a/Program.cs +++ b/Program.cs @@ -2,149 +2,180 @@ using System.Linq; using System.Diagnostics; using System.Text.RegularExpressions; +using ttrss_co_client.tasks; namespace ttrss_co_client { class Program { + private static ttrss.ApiClient ttrssClient; static async Task Main(string[] args) { + + // var tasktypes = AppDomain.CurrentDomain.GetAssemblies() + // .SelectMany(domainAssembly => domainAssembly.GetTypes()) + // .Where(type => type.IsSubclassOf(typeof(Phase1Task)) && !type.IsAbstract) + // .ToList(); + // foreach (var subtype in subtypes) + // { + // var inst = (CoClientTask) Activator.CreateInstance(subtype); + // } + var conf = Configure(); Console.WriteLine($"{DateTime.Now.ToLongTimeString()}"); - var ttrssClient = new ttrss.ApiClient(conf.BaseURI); + ttrssClient = new ttrss.ApiClient(conf.BaseURI); await ttrssClient.Login(conf.Username, conf.Password); var loggedin = await ttrssClient.IsLoggedIn(); Console.WriteLine($"logged in: {loggedin}"); + + #region phase 1 + Console.WriteLine("===== phase 1 ====="); + + var Phase1Tasks = new List>(); + + Phase1Task.TtrssClient = ttrssClient; + Phase1Task.Conf = conf; + var phase1TaskTypes = AppDomain.CurrentDomain.GetAssemblies() + .SelectMany(domainAssembly => domainAssembly.GetTypes()) + .Where(type => type.IsSubclassOf(typeof(Phase1Task)) && !type.IsAbstract) + .ToList(); + var phase1TaskConcretions = new List(); + foreach (var phase1TaskType in phase1TaskTypes) + { + var concretion = (Phase1Task) Activator.CreateInstance(phase1TaskType); + concretion.TriggerLabel = conf.feedActions.FirstOrDefault(fa => fa.command == concretion.TaskName).triggerlabelCaption; + phase1TaskConcretions.Add(concretion); + } + Console.WriteLine($"{phase1TaskConcretions.Count()} phase 1 task types understood"); + var unreadFeeds = await ttrssClient.GetFeeds(cat_id: -3, unread_only: true); + Console.WriteLine($"{unreadFeeds.Count()} feeds unread"); foreach (var uf in unreadFeeds) { Console.WriteLine($"unread feed: {uf.title}"); var headlines = await ttrssClient.GetHeadlines(uf.id, view_mode: ttrss.ApiClient.VIEWMODE.Unread, include_attachments: true); foreach (var hl in headlines) { - var labelsWRTFeed = (await ttrssClient.GetLabels(hl.id)); + var labelsWRTArticle = (await ttrssClient.GetLabels(hl.id)); var actionsForFeed = conf.feedActions.Where(fa => - labelsWRTFeed.Where(l => l.@checked).Select(l => l.caption).Contains(fa.triggerlabelCaption))?.ToList(); + labelsWRTArticle.Where(l => l.@checked).Select(l => l.caption).Contains(fa.triggerlabelCaption))?.ToList(); if (actionsForFeed != null && actionsForFeed.Any()) { - var sponsorCheck = await SponsorCheck(hl); - if (!sponsorCheck.Item1) + var action = actionsForFeed.First(); + + Console.WriteLine($" headline {hl.title} has label {action.triggerlabelCaption} / action {action.command}"); + + var appropriatePhase1Task = phase1TaskConcretions.FirstOrDefault(tt => tt.TaskName == action.command); + if(appropriatePhase1Task != null) { - await ttrssClient.UpdateArticleNote($"{hl.note}\n[{DateTime.Now.ToLongTimeString()}] sponsorcheck: {sponsorCheck.Item2}", hl.id); - continue; + Phase1Tasks.Add(appropriatePhase1Task.ActOn(hl, labelsWRTArticle)); } - foreach (var action in actionsForFeed) + else { - Console.WriteLine($" {hl.title} -> action: {action.command}"); - var noteString = $"{hl.note}\n[{DateTime.Now.ToLongTimeString()}] sponsorcheck: {sponsorCheck.Item2}"; - ttrss.datastructures.Label nameLabel; - string podcastName; - if (!string.IsNullOrWhiteSpace(noteString)) - { - noteString += $"{hl.note}\n"; - } - switch (action.command) - { - case "dl": - var stdDLResult = await standardDL(hl.link.ToString()); - await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToLongTimeString()}] - {stdDLResult.Item2}", hl.id); - if (stdDLResult.Item1 == true) - { - Console.WriteLine($" {hl.title} -> dl success, removing label"); - await ttrssClient.SetArticleLabel( - labelsWRTFeed.First(l => l.caption == action.triggerlabelCaption).id, - false, - hl.id); - } - else - { - Console.WriteLine($" {hl.title} -> dl failed"); - } - break; - case "podcastifyYT": - nameLabel = labelsWRTFeed.FirstOrDefault(l => l.caption?.StartsWith(conf.PodcastTitlePrefix) == true && l.@checked); - podcastName = nameLabel?.caption.Substring(conf.PodcastTitlePrefix.Length) - ?? hl.feed_title; - var YTpodcastifyResult = await podcastifyYT(hl.link.ToString(), podcastName); - await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToLongTimeString()}] - {YTpodcastifyResult.Item2}", hl.id); - if (YTpodcastifyResult.Item1 == true) - { - Console.WriteLine($" {hl.title} -> podcastify (YT) success, removing labels"); - await ttrssClient.SetArticleLabel( - labelsWRTFeed.First(l => l.caption == action.triggerlabelCaption).id, - false, - hl.id); - if (nameLabel != null) - { - await ttrssClient.SetArticleLabel(nameLabel.id, false, hl.id); - } - } - else - { - Console.WriteLine($" {hl.title} -> podcastify (YT) failed"); - } - break; - case "podcastifyAttachment": - nameLabel = labelsWRTFeed.FirstOrDefault(l => l.caption?.StartsWith(conf.PodcastTitlePrefix) == true && l.@checked); - podcastName = nameLabel?.caption.Substring(conf.PodcastTitlePrefix.Length) - ?? hl.feed_title; - var attachmentLink = hl.attachments.Select(a => a.content_url)?.FirstOrDefault(); - var ATTpodcastifyResult = await podcastifyAttachment(attachmentLink.ToString(), podcastName, hl.title); - await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToLongTimeString()}] - {ATTpodcastifyResult.Item2}", hl.id); - if (ATTpodcastifyResult.Item1 == true) - { - Console.WriteLine($" {hl.title} -> podcastify (att) success, removing labels"); - await ttrssClient.SetArticleLabel( - labelsWRTFeed.First(l => l.caption == action.triggerlabelCaption).id, - false, - hl.id); - if (nameLabel != null) - { - await ttrssClient.SetArticleLabel(nameLabel.id, false, hl.id); - } - } - else - { - Console.WriteLine($" {hl.title} -> podcastify (att) failed"); - } - break; - default: - noteString += $"[{DateTime.Now.ToLongTimeString()}] - feed configured but action not recognized"; - break; - } + Console.Error.WriteLine($"couldn't find phase 1 task {action.command} for workorder referring to article id {hl.id}!"); } } } } - - - Console.WriteLine($"logging out"); - await ttrssClient.Logout(); - Console.WriteLine($"done, moving files from temporary location"); - - var resulted = Directory.GetFiles("tmp", "*.*", SearchOption.AllDirectories); - if (resulted.Count() > 0) + Console.WriteLine($"done processing feeds. phase 1 tasks launched. wait to complete."); + var remainingWork = new List(); + foreach(var lingeringTask in Phase1Tasks) { - foreach (var f in resulted) + var wo = await lingeringTask; + if(wo != null) { - var moveTarget = Path.Combine(conf.OnDoneCopy, f.Substring("tmp/".Length)); - if (!Path.Exists(Path.GetDirectoryName(moveTarget))) + Console.WriteLine($"articleId {wo.articleId} left a work order; it has {wo.Phase2TaskList.Count()} phase 2 task{(wo.Phase2TaskList.Count() == 1 ? "" : "s")}"); + remainingWork.Add(wo); + } + } + Console.WriteLine($"phase 1 tasks complete, carrying {remainingWork.Count()} workorder{(remainingWork.Count() == 1 ? "" : "s")} to phase 2"); + #endregion + + #region local tasks (stripping ads, converting.) + Console.WriteLine("===== phase 2 ====="); + //loop through working directory looking for other work orders to add + var workOrderFiles = Directory.GetFiles($"{conf.WorkingDirectory}", "workorder.json", new EnumerationOptions(){RecurseSubdirectories = true}); + Console.WriteLine($"{workOrderFiles.Count()} workorder file{(workOrderFiles.Count() == 1 ? "" : "s")} located"); + foreach(var workorderFile in workOrderFiles) + { + try + { + remainingWork.Add(JsonConvert.DeserializeObject(workorderFile)); + Console.WriteLine($"picked up workorder task; {workorderFile}"); + } + catch(Exception e) + { + Console.Error.WriteLine($"error picking up work order file {workorderFile} - {e.Message}; {e.StackTrace}"); + } + } + Console.WriteLine($"{remainingWork.Count} phase 2 workorders, between pulled from ttrss and read from files"); + + while(remainingWork.Count > 0) + { + //todo: solve the halting problem + //ok but seriously, + Phase2Task.TtrssClient = ttrssClient; + Phase2Task.Conf = conf; + var phase2TaskTypes = AppDomain.CurrentDomain.GetAssemblies() + .SelectMany(domainAssembly => domainAssembly.GetTypes()) + .Where(type => type.IsSubclassOf(typeof(Phase2Task)) && !type.IsAbstract) + .ToList(); + var phase2TaskConcretions = new List(); + foreach (var phase2TaskType in phase2TaskTypes) + { + var concretion = (Phase2Task) Activator.CreateInstance(phase2TaskType); + phase2TaskConcretions.Add(concretion); + } + Console.WriteLine($"{phase2TaskConcretions.Count()} phase 2 task types understood"); + + var Phase2Tasks = new List>>(); + + + Console.WriteLine($"launching first pass over work orders in phase 2."); + foreach(var wo in remainingWork) + { + var taskName = wo.Phase2TaskList[wo.Phase2TaskList.Keys.Min()]; + + var appropriatePhase2Task = phase2TaskConcretions.FirstOrDefault(tt => tt.TaskName == taskName); + if(appropriatePhase2Task != null) { - Directory.CreateDirectory(Path.GetDirectoryName(moveTarget)); - } - if (File.Exists(moveTarget)) - { - Console.WriteLine("file already exists, abandoning"); + wo.Phase2TaskList.Remove(wo.Phase2TaskList.Keys.Min()); + Phase2Tasks.Add(appropriatePhase2Task.ActOn(wo)); } else { - File.Move(f, moveTarget); + Console.Error.WriteLine($"couldn't find phase 2 task {taskName} for workorder referring to article id {wo.articleId}!"); } } + + Console.WriteLine($"phase 2 tasks launched. now the complex part."); + + remainingWork = new List(); + foreach(var lingeringTask in Phase2Tasks) + { + var wo = await lingeringTask; + switch (wo.Item1) + { + case Phase2Task.TaskStatus.Done: + // :) + break; + case Phase2Task.TaskStatus.ContinueNow: + remainingWork.Add(wo.Item2); + break; + case Phase2Task.TaskStatus.TryLater: + File.WriteAllText(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString(), "workorder.json"), JsonConvert.SerializeObject(wo)); + break; + } + } + Console.WriteLine($"{remainingWork.Count} phase 2 tasks not complete - should be picked up next run.."); } +#endregion + await ttrssClient.Logout(); + Console.WriteLine($"logged out of ttrss."); + Console.WriteLine($"done for real"); } static Configuration Configure(string configurationPath = "appsettings.json") @@ -174,49 +205,6 @@ namespace ttrss_co_client return conf; } - - private static async Task> standardDL(string articleLink) - { - Console.WriteLine($" standard downloading {articleLink}"); - try - { - var ytdl = new YoutubeDLSharp.YoutubeDL(); - ytdl.YoutubeDLPath = "yt-dlp"; - ytdl.FFmpegPath = "ffmpeg"; - ytdl.OutputFolder = "./tmp/recent episodes"; - ytdl.OutputFileTemplate = "%(upload_date)s - %(title)s - [%(id)s].%(ext)s"; - var sw = new Stopwatch(); - sw.Start(); - var res = await ytdl.RunVideoDownload(articleLink); - sw.Stop(); - var outputStr = $"{(res.Success ? "Success" : "fail")} in {sw.Elapsed}"; - if (res.ErrorOutput != null && res.ErrorOutput.Length > 0) - { - outputStr += "\n" + string.Join('\n', res.ErrorOutput); - } - Console.WriteLine($" download {(res.Success ? "success" : "failed")}: {articleLink} -> {res.Data}"); - if (!res.Data.EndsWith(".mp4")) - { - Console.WriteLine(" must convert video"); - sw.Reset(); - var outputFilename = res.Data.Substring(0, res.Data.LastIndexOf('.')) + ".mp4"; - sw.Start(); - var conversionProc = Process.Start("ffmpeg", $"-y -i \"{res.Data}\" \"{outputFilename}\""); - conversionProc.WaitForExit(); - sw.Stop(); - Console.WriteLine($" converted {res.Data} -> {outputFilename}"); - File.Delete(res.Data); - outputStr += $"\nconverted in {sw.Elapsed}"; - } - return new Tuple(res.Success, outputStr); - - } - catch (Exception e) - { - Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}"); - return new Tuple(false, $"{e.ToString()}: {e.Message}.\n{e.StackTrace}"); - } - } private static async Task> podcastifyYT(string articleLink, string podcastName) { Console.WriteLine($" youtube-podcastifying {articleLink} ({podcastName})"); @@ -296,50 +284,50 @@ namespace ttrss_co_client return new Tuple(false, $"{e.ToString()}: {e.Message}.\n{e.StackTrace}"); } } - private static async Task> SponsorCheck(ttrss.datastructures.Headline hl) - { - if (!hl.link.Host.EndsWith("youtube.com")) - { - return new Tuple(true, "sponsorblock, sadly, only exists for youtube"); - } - var match = Regex.Match(hl.link.Query, "v=([^&]+)(&|$)"); - var videoId = match.Groups?[1].Value; - var c = new HttpClient(); + // private static async Task> SponsorCheck(ttrss.datastructures.Headline hl) + // { + // if (!hl.link.Host.EndsWith("youtube.com")) + // { + // return new Tuple(true, "sponsorblock, sadly, only exists for youtube"); + // } + // var match = Regex.Match(hl.link.Query, "v=([^&]+)(&|$)"); + // var videoId = match.Groups?[1].Value; + // var c = new HttpClient(); - var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview"); - if (sponsorblockcheck.StatusCode == System.Net.HttpStatusCode.NotFound) - { - Console.WriteLine($"sponsorblock reports that {videoId} has no entries (yet)"); - var updateTimestamp = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); - updateTimestamp = updateTimestamp.AddSeconds(hl.updated).ToLocalTime(); - if (DateTime.Now - updateTimestamp > TimeSpan.FromMinutes(45)) - { - return new Tuple(true, $"updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock"); - } - else - { - return new Tuple(false, "none found, waiting"); - } - } - else - { - try - { - var segments = JsonConvert.DeserializeObject>(await sponsorblockcheck.Content.ReadAsStringAsync()); - if (segments.Count() > 1) - { - return new Tuple(true, $"{segments.Count()} segments"); - } - else - { - return new Tuple(false, $"no segments"); - } - } - catch (Exception e) - { - return new Tuple(false, $"{e.ToString()} - {e.Message}"); - } - } - } + // var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview"); + // if (sponsorblockcheck.StatusCode == System.Net.HttpStatusCode.NotFound) + // { + // Console.WriteLine($"sponsorblock reports that {videoId} has no entries (yet)"); + // var updateTimestamp = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + // updateTimestamp = updateTimestamp.AddSeconds(hl.updated).ToLocalTime(); + // if (DateTime.Now - updateTimestamp > TimeSpan.FromMinutes(45)) + // { + // return new Tuple(true, $"updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock"); + // } + // else + // { + // return new Tuple(false, "none found, waiting"); + // } + // } + // else + // { + // try + // { + // var segments = JsonConvert.DeserializeObject>(await sponsorblockcheck.Content.ReadAsStringAsync()); + // if (segments.Count() > 1) + // { + // return new Tuple(true, $"{segments.Count()} segments"); + // } + // else + // { + // return new Tuple(false, $"no segments"); + // } + // } + // catch (Exception e) + // { + // return new Tuple(false, $"{e.ToString()} - {e.Message}"); + // } + // } + // } } } \ No newline at end of file diff --git a/sample-appsettings.json b/sample-appsettings.json index 2a46a81..c86bab2 100644 --- a/sample-appsettings.json +++ b/sample-appsettings.json @@ -4,6 +4,7 @@ "password": "sordph1sh", "podcastTitlePrefix": "[podcast title] - ", "onDoneCopy":"./", + "workingDirectory":"working/", "feedActions": [ { diff --git a/tasks/Convert.cs b/tasks/Convert.cs new file mode 100644 index 0000000..b4adec3 --- /dev/null +++ b/tasks/Convert.cs @@ -0,0 +1,15 @@ +// using System.Diagnostics; +// using ttrss_co_client.ttrss; +// using ttrss_co_client.ttrss.datastructures; + +// namespace ttrss_co_client.tasks +// { +// ///ffmpegify +// public class Convert : Phase2Task +// { +// public override async Task ActOn(WorkOrder wo) +// { + +// } +// } +// } \ No newline at end of file diff --git a/tasks/Phase1Task.cs b/tasks/Phase1Task.cs new file mode 100644 index 0000000..d9385de --- /dev/null +++ b/tasks/Phase1Task.cs @@ -0,0 +1,15 @@ +using ttrss_co_client.ttrss; +using ttrss_co_client.ttrss.datastructures; + +namespace ttrss_co_client.tasks +{ + ///generally, download + public abstract class Phase1Task + { + public virtual string TaskName { get { return this.GetType().ToString(); } } + public static ApiClient TtrssClient { get; set; } + public static Configuration Conf { get; set; } + public virtual string TriggerLabel { get; set; } + public abstract Task ActOn(Headline headline, IEnumerable