From 3fcc5f376df6a4693cd3b88c063aedd58e5d8813 Mon Sep 17 00:00:00 2001 From: Adam R Grey Date: Sun, 9 Apr 2023 17:01:25 -0400 Subject: [PATCH] IT WORKS (again) (almost) standard DL downloads, converts first (maybe that's bad?), strips ads :) --- Program.cs | 82 +++++++++++++++-------------------- tasks/Convert.cs | 2 +- tasks/PodcastifyAttachment.cs | 6 +-- tasks/PodcastifyYT.cs | 6 +-- tasks/Publish.cs | 19 ++++---- tasks/Sponsorblock.cs | 48 +++++++++++++------- tasks/StandardDL.cs | 25 ++++++----- 7 files changed, 99 insertions(+), 89 deletions(-) diff --git a/Program.cs b/Program.cs index 6a83f51..3a023f4 100644 --- a/Program.cs +++ b/Program.cs @@ -11,18 +11,8 @@ namespace ttrss_co_client private static ttrss.ApiClient ttrssClient; static async Task Main(string[] args) { - - // var tasktypes = AppDomain.CurrentDomain.GetAssemblies() - // .SelectMany(domainAssembly => domainAssembly.GetTypes()) - // .Where(type => type.IsSubclassOf(typeof(Phase1Task)) && !type.IsAbstract) - // .ToList(); - // foreach (var subtype in subtypes) - // { - // var inst = (CoClientTask) Activator.CreateInstance(subtype); - // } - var conf = Configure(); - Console.WriteLine($"{DateTime.Now.ToLongTimeString()}"); + Console.WriteLine($"{DateTime.Now.ToString("o")}"); ttrssClient = new ttrss.ApiClient(conf.BaseURI); await ttrssClient.Login(conf.Username, conf.Password); @@ -30,7 +20,6 @@ namespace ttrss_co_client var loggedin = await ttrssClient.IsLoggedIn(); Console.WriteLine($"logged in: {loggedin}"); - #region phase 1 Console.WriteLine("===== phase 1 ====="); @@ -43,12 +32,12 @@ namespace ttrss_co_client .Where(type => type.IsSubclassOf(typeof(Phase1Task)) && !type.IsAbstract) .ToList(); var phase1TaskConcretions = new List(); - foreach (var phase1TaskType in phase1TaskTypes) - { - var concretion = (Phase1Task) Activator.CreateInstance(phase1TaskType); - concretion.TriggerLabel = conf.feedActions.FirstOrDefault(fa => fa.command == concretion.TaskName).triggerlabelCaption; - phase1TaskConcretions.Add(concretion); - } + foreach (var phase1TaskType in phase1TaskTypes) + { + var concretion = (Phase1Task)Activator.CreateInstance(phase1TaskType); + concretion.TriggerLabel = conf.feedActions.FirstOrDefault(fa => fa.command == concretion.TaskName).triggerlabelCaption; + phase1TaskConcretions.Add(concretion); + } Console.WriteLine($"{phase1TaskConcretions.Count()} phase 1 task types understood"); var unreadFeeds = await ttrssClient.GetFeeds(cat_id: -3, unread_only: true); @@ -65,11 +54,11 @@ namespace ttrss_co_client if (actionsForFeed != null && actionsForFeed.Any()) { var action = actionsForFeed.First(); - + Console.WriteLine($" headline {hl.title} has label {action.triggerlabelCaption} / action {action.command}"); var appropriatePhase1Task = phase1TaskConcretions.FirstOrDefault(tt => tt.TaskName == action.command); - if(appropriatePhase1Task != null) + if (appropriatePhase1Task != null) { Phase1Tasks.Add(appropriatePhase1Task.ActOn(hl, labelsWRTArticle)); } @@ -82,10 +71,10 @@ namespace ttrss_co_client } Console.WriteLine($"done processing feeds. phase 1 tasks launched. wait to complete."); var remainingWork = new List(); - foreach(var lingeringTask in Phase1Tasks) + foreach (var lingeringTask in Phase1Tasks) { var wo = await lingeringTask; - if(wo != null) + if (wo != null) { Console.WriteLine($"articleId {wo.articleId} left a work order; it has {wo.Phase2TaskList.Count()} phase 2 task{(wo.Phase2TaskList.Count() == 1 ? "" : "s")}"); remainingWork.Add(wo); @@ -97,16 +86,17 @@ namespace ttrss_co_client #region local tasks (stripping ads, converting.) Console.WriteLine("===== phase 2 ====="); //loop through working directory looking for other work orders to add - var workOrderFiles = Directory.GetFiles($"{conf.WorkingDirectory}", "workorder.json", new EnumerationOptions(){RecurseSubdirectories = true}); + var workOrderFiles = Directory.GetFiles($"{conf.WorkingDirectory}", "workorder.json", new EnumerationOptions() { RecurseSubdirectories = true }); Console.WriteLine($"{workOrderFiles.Count()} workorder file{(workOrderFiles.Count() == 1 ? "" : "s")} located"); - foreach(var workorderFile in workOrderFiles) + foreach (var workorderFile in workOrderFiles) { try { - remainingWork.Add(JsonConvert.DeserializeObject(workorderFile)); + remainingWork.Add(JsonConvert.DeserializeObject(File.ReadAllText(workorderFile))); Console.WriteLine($"picked up workorder task; {workorderFile}"); + File.Delete(workorderFile); } - catch(Exception e) + catch (Exception e) { Console.Error.WriteLine($"error picking up work order file {workorderFile} - {e.Message}; {e.StackTrace}"); } @@ -122,26 +112,26 @@ namespace ttrss_co_client var phase2TaskConcretions = new List(); foreach (var phase2TaskType in phase2TaskTypes) { - var concretion = (Phase2Task) Activator.CreateInstance(phase2TaskType); + var concretion = (Phase2Task)Activator.CreateInstance(phase2TaskType); phase2TaskConcretions.Add(concretion); } - while(remainingWork.Count > 0) + while (remainingWork.Count > 0) { //todo: solve the halting problem //ok but seriously, maybe I could time out work orders after a while? but converting a 3-hour 4k video could easily take an hour! Console.WriteLine($"{phase2TaskConcretions.Count()} phase 2 task types understood"); var Phase2Tasks = new List>>(); - - + + Console.WriteLine($"launching first pass over work orders in phase 2."); - foreach(var wo in remainingWork) + foreach (var wo in remainingWork) { var taskName = wo.Phase2TaskList[wo.Phase2TaskList.Keys.Min()]; - + var appropriatePhase2Task = phase2TaskConcretions.FirstOrDefault(tt => tt.TaskName == taskName); - if(appropriatePhase2Task != null) + if (appropriatePhase2Task != null) { wo.Phase2TaskList.Remove(wo.Phase2TaskList.Keys.Min()); Phase2Tasks.Add(appropriatePhase2Task.ActOn(wo)); @@ -153,32 +143,32 @@ namespace ttrss_co_client } Console.WriteLine($"phase 2 tasks launched. now the complex part."); - + remainingWork = new List(); - foreach(var lingeringTask in Phase2Tasks) + foreach (var lingeringTask in Phase2Tasks) { var wo = await lingeringTask; - //if you tell me it's done, or you need to continue now.... I believe you. - //TODO: be smart enough to override? + //if you tell me it's done, or you need to continue now.... I believe you. + //TODO: be smart enough to override? switch (wo.Item1) { case Phase2Task.TaskStatus.Done: - // :) - break; + Directory.Delete(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString())); + break; case Phase2Task.TaskStatus.ContinueNow: remainingWork.Add(wo.Item2); - break; + break; case Phase2Task.TaskStatus.TryLater: - File.WriteAllText(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString(), "workorder.json"), JsonConvert.SerializeObject(wo)); - break; + File.WriteAllText(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString(), "workorder.json"), JsonConvert.SerializeObject(wo.Item2)); + break; } } Console.WriteLine($"{remainingWork.Count} phase 2 tasks to be re-looped."); } -#endregion - await ttrssClient.Logout(); - Console.WriteLine($"logged out of ttrss."); - + #endregion + await ttrssClient.Logout(); + Console.WriteLine($"logged out of ttrss."); + Console.WriteLine($"done for real"); } static Configuration Configure(string configurationPath = "appsettings.json") diff --git a/tasks/Convert.cs b/tasks/Convert.cs index e5a659c..1c51fd3 100644 --- a/tasks/Convert.cs +++ b/tasks/Convert.cs @@ -17,7 +17,7 @@ namespace ttrss_co_client.tasks } var sw = new Stopwatch(); sw.Start(); - var conversionProc = Process.Start("ffmpeg", $"-y -i \"{wo.data["path"]}\" \"{wo.data["conversion-target"]}\""); + var conversionProc = Process.Start("ffmpeg", $"-y -loglevel quiet -i \"{wo.data["path"]}\" \"{wo.data["conversion-target"]}\""); conversionProc.WaitForExit(); sw.Stop(); if(File.Exists(wo.data["conversion-target"])) diff --git a/tasks/PodcastifyAttachment.cs b/tasks/PodcastifyAttachment.cs index 52a24e2..6224d54 100644 --- a/tasks/PodcastifyAttachment.cs +++ b/tasks/PodcastifyAttachment.cs @@ -17,7 +17,7 @@ // // ?? hl.feed_title; // // var attachmentLink = hl.attachments.Select(a => a.content_url)?.FirstOrDefault(); // // var ATTpodcastifyResult = await podcastifyAttachment(attachmentLink.ToString(), podcastName, hl.title); -// // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToLongTimeString()}] - {ATTpodcastifyResult.Item2}", hl.id); +// // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToString("o")}] - {ATTpodcastifyResult.Item2}", hl.id); // // if (ATTpodcastifyResult.Item1 == true) // // { // // Console.WriteLine($" {hl.title} -> podcastify (att) success, removing labels"); @@ -62,7 +62,7 @@ // // Console.WriteLine($" {headline.title}: label removed"); // // if (!res.Success) // // { -// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id); +// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id); // // return null; // // } // // else @@ -93,7 +93,7 @@ // // { // // Console.Error.WriteLine($"fatal error in standard DL for {headline.link}"); // // Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}"); -// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id); +// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id); // // return null; // // } diff --git a/tasks/PodcastifyYT.cs b/tasks/PodcastifyYT.cs index 851a845..dc15a42 100644 --- a/tasks/PodcastifyYT.cs +++ b/tasks/PodcastifyYT.cs @@ -15,7 +15,7 @@ // // podcastName = nameLabel?.caption.Substring(conf.PodcastTitlePrefix.Length) // // ?? hl.feed_title; // // var YTpodcastifyResult = await podcastifyYT(hl.link.ToString(), podcastName); -// // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToLongTimeString()}] - {YTpodcastifyResult.Item2}", hl.id); +// // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToString("o")}] - {YTpodcastifyResult.Item2}", hl.id); // // if (YTpodcastifyResult.Item1 == true) // // { // // Console.WriteLine($" {hl.title} -> podcastify (YT) success, removing labels"); @@ -59,7 +59,7 @@ // // Console.WriteLine($" {headline.title}: label removed"); // // if (!res.Success) // // { -// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id); +// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id); // // return null; // // } // // else @@ -90,7 +90,7 @@ // // { // // Console.Error.WriteLine($"fatal error in standard DL for {headline.link}"); // // Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}"); -// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id); +// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id); // // return null; // // } diff --git a/tasks/Publish.cs b/tasks/Publish.cs index 27db26e..771097f 100644 --- a/tasks/Publish.cs +++ b/tasks/Publish.cs @@ -11,16 +11,15 @@ namespace ttrss_co_client.tasks public override string TaskName => "filemovePublish"; public override async Task>ActOn(WorkOrder workOrder) { - var wo = workOrder as PublishWorkOrder; - File.Move(wo.Path, wo.PublishTarget, true); - var article = (await TtrssClient.GetArticles(wo.articleId))?.FirstOrDefault(); - await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToShortDateString()}] - copied"); - return new Tuple(TaskStatus.Done, wo); + var targetDirectory = Path.GetDirectoryName(workOrder.data["publish-target"]); + if(!Directory.Exists(targetDirectory)) + { + Directory.CreateDirectory(targetDirectory); + } + File.Move(workOrder.data["path"], workOrder.data["publish-target"], true); + var article = (await TtrssClient.GetArticles(workOrder.articleId))?.FirstOrDefault(); + await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToShortDateString()}] - copied", article.id); + return new Tuple(TaskStatus.Done, workOrder); } } - public class PublishWorkOrder : WorkOrder - { - public string Path => data["path"]; - public string PublishTarget => data["publish-target"]; - } } \ No newline at end of file diff --git a/tasks/Sponsorblock.cs b/tasks/Sponsorblock.cs index e6cc387..1087b56 100644 --- a/tasks/Sponsorblock.cs +++ b/tasks/Sponsorblock.cs @@ -1,3 +1,5 @@ +using System; +using System.Text; using Newtonsoft.Json; using System.Linq; using System.Diagnostics; @@ -30,7 +32,7 @@ namespace ttrss_co_client.tasks var videoId = match.Groups?[1].Value; var c = new HttpClient(); - var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview"); + var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview&category=interaction"); IEnumerable segments = null; try { @@ -47,18 +49,19 @@ namespace ttrss_co_client.tasks updateTimestamp = updateTimestamp.AddSeconds(article.updated).ToLocalTime(); if (DateTime.Now - updateTimestamp > TimeSpan.FromMinutes(45)) { - await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock", article.id); + await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock", article.id); return new Tuple(TaskStatus.ContinueNow, workOrder); } else { - await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] waiting for sponsorblock segments", article.id); + await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] waiting for sponsorblock segments", article.id); workOrder.Phase2TaskList[workOrder.Phase2TaskList.Keys.Min() - 1] = this.TaskName; return new Tuple(TaskStatus.TryLater, workOrder); } } else { + Console.WriteLine($"[{DateTime.Now.ToString("o")}] sponsorblock reports {segments.Count()} junk segments"); var contentSegments = new List>(); var previousEdge = 0.0; var extension = workOrder.data["path"].Substring(workOrder.data["path"].LastIndexOf('.')); @@ -76,37 +79,50 @@ namespace ttrss_co_client.tasks } contentSegments.Add(new Tuple(previousEdge, segments.First().videoDuration)); + #region ffmpeg via intermediate files var intermediateCount = 0; foreach(var seg in contentSegments) { - var intermediateTargetPath = $"{intermediatePathBase}-intermediate-{intermediateCount.ToString("ddd")}{extension}"; - conversionProcesses.Add(new Tuple>( - Process.Start("ffmpeg", $"-y -i \"{workOrder.data["path"]}\" \"{intermediateTargetPath}\""), - new Tuple(intermediateCount, intermediateTargetPath) - )); + var intermediateTargetPath = $"{intermediatePathBase}-intermediate-{intermediateCount.ToString("D2")}{extension}"; + conversionProcesses.Add(new Tuple>( + Process.Start("ffmpeg", $"-y -loglevel quiet -i \"{workOrder.data["path"]}\" -ss {seg.Item1} -to {seg.Item2} \"{intermediateTargetPath}\""), + new Tuple(intermediateCount, intermediateTargetPath) + )); + Console.WriteLine("waiting for exit from task, for debugging reasons"); + conversionProcesses.Last().Item1.WaitForExit(); intermediateCount++; } - Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments being exported"); + Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments being exported"); var intermediates = new Dictionary(); foreach(var proc in conversionProcesses) { proc.Item1.WaitForExit(); intermediates[proc.Item2.Item1] = proc.Item2.Item2; } - Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments should be exported, stitching together"); - - Process.Start("ffmpeg", $"-y -i \"concat:{string.Join('|', intermediates.OrderBy(kvp => kvp.Key).Select(kvp => kvp.Value).ToArray())}\" -c copy \"{workOrder.data["path"]}\"") - .WaitForExit(); + Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments should be exported, stitching together"); - Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments stitched. Deleting originals."); + var sb = new StringBuilder(); + sb.AppendLine("#ffmpeg demands it"); + foreach(var intermediate in intermediates.OrderBy(kvp => kvp.Key)) + { + sb.AppendLine($"file '{intermediate.Value}'"); + } + var ffmpegFile = Path.Combine(Path.GetDirectoryName(workOrder.data["path"]), "ffmpeglist.txt"); + File.WriteAllText(ffmpegFile, sb.ToString()); + + Process.Start("ffmpeg", $"-y -f concat -safe 0 -i {ffmpegFile} -c copy \"{workOrder.data["path"]}\"") + .WaitForExit(); + File.Delete(ffmpegFile); + + Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments stitched. Deleting originals."); foreach(var intermediate in intermediates.Values) { File.Delete(intermediate); } - + #endregion sw.Stop(); - await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] removed {segments.Count()} junk segments", article.id); + await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] removed {segments.Count()} junk segments", article.id); return new Tuple(TaskStatus.ContinueNow, workOrder); } diff --git a/tasks/StandardDL.cs b/tasks/StandardDL.cs index 64c2ec3..5e055cd 100644 --- a/tasks/StandardDL.cs +++ b/tasks/StandardDL.cs @@ -11,13 +11,17 @@ namespace ttrss_co_client.tasks public override async Task ActOn(Headline headline, IEnumerable