adam/ttrss-co-pilot
adam/ttrss-co-pilot
1
0
Fork 0
Code Issues 6 Pull Requests Packages Projects Releases 7 Wiki Activity

IT WORKS (again) (almost)

Browse Source 
standard DL downloads, converts first (maybe that's bad?), strips ads :)
This commit is contained in:
Adam R Grey 2023-04-09 17:01:25 -04:00
parent a64e646b04
commit 3fcc5f376d
7 changed files with 99 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
Program.cs
View File

@ -11,18 +11,8 @@ namespace ttrss_co_client
private static ttrss.ApiClient ttrssClient; private static ttrss.ApiClient ttrssClient;
static async Task Main(string[] args) static async Task Main(string[] args)
{ {
// var tasktypes = AppDomain.CurrentDomain.GetAssemblies()
// .SelectMany(domainAssembly => domainAssembly.GetTypes())
// .Where(type => type.IsSubclassOf(typeof(Phase1Task)) && !type.IsAbstract)
// .ToList();
// foreach (var subtype in subtypes)
// {
// var inst = (CoClientTask) Activator.CreateInstance(subtype);
// }
var conf = Configure(); var conf = Configure();
Console.WriteLine($"{DateTime.Now.ToLongTimeString()}"); Console.WriteLine($"{DateTime.Now.ToString("o")}");
ttrssClient = new ttrss.ApiClient(conf.BaseURI); ttrssClient = new ttrss.ApiClient(conf.BaseURI);
await ttrssClient.Login(conf.Username, conf.Password); await ttrssClient.Login(conf.Username, conf.Password);
@ -30,7 +20,6 @@ namespace ttrss_co_client
var loggedin = await ttrssClient.IsLoggedIn(); var loggedin = await ttrssClient.IsLoggedIn();
Console.WriteLine($"logged in: {loggedin}"); Console.WriteLine($"logged in: {loggedin}");
#region phase 1 #region phase 1
Console.WriteLine("===== phase 1 ====="); Console.WriteLine("===== phase 1 =====");
@ -43,12 +32,12 @@ namespace ttrss_co_client
.Where(type => type.IsSubclassOf(typeof(Phase1Task)) && !type.IsAbstract) .Where(type => type.IsSubclassOf(typeof(Phase1Task)) && !type.IsAbstract)
.ToList(); .ToList();
var phase1TaskConcretions = new List<Phase1Task>(); var phase1TaskConcretions = new List<Phase1Task>();
foreach (var phase1TaskType in phase1TaskTypes) foreach (var phase1TaskType in phase1TaskTypes)
{ {
var concretion = (Phase1Task) Activator.CreateInstance(phase1TaskType); var concretion = (Phase1Task)Activator.CreateInstance(phase1TaskType);
concretion.TriggerLabel = conf.feedActions.FirstOrDefault(fa => fa.command == concretion.TaskName).triggerlabelCaption; concretion.TriggerLabel = conf.feedActions.FirstOrDefault(fa => fa.command == concretion.TaskName).triggerlabelCaption;
phase1TaskConcretions.Add(concretion); phase1TaskConcretions.Add(concretion);
} }
Console.WriteLine($"{phase1TaskConcretions.Count()} phase 1 task types understood"); Console.WriteLine($"{phase1TaskConcretions.Count()} phase 1 task types understood");
var unreadFeeds = await ttrssClient.GetFeeds(cat_id: -3, unread_only: true); var unreadFeeds = await ttrssClient.GetFeeds(cat_id: -3, unread_only: true);
@ -65,11 +54,11 @@ namespace ttrss_co_client
if (actionsForFeed != null && actionsForFeed.Any()) if (actionsForFeed != null && actionsForFeed.Any())
{ {
var action = actionsForFeed.First(); var action = actionsForFeed.First();
Console.WriteLine($" headline {hl.title} has label {action.triggerlabelCaption} / action {action.command}"); Console.WriteLine($" headline {hl.title} has label {action.triggerlabelCaption} / action {action.command}");
var appropriatePhase1Task = phase1TaskConcretions.FirstOrDefault(tt => tt.TaskName == action.command); var appropriatePhase1Task = phase1TaskConcretions.FirstOrDefault(tt => tt.TaskName == action.command);
if(appropriatePhase1Task != null) if (appropriatePhase1Task != null)
{ {
Phase1Tasks.Add(appropriatePhase1Task.ActOn(hl, labelsWRTArticle)); Phase1Tasks.Add(appropriatePhase1Task.ActOn(hl, labelsWRTArticle));
} }
@ -82,10 +71,10 @@ namespace ttrss_co_client
} }
Console.WriteLine($"done processing feeds. phase 1 tasks launched. wait to complete."); Console.WriteLine($"done processing feeds. phase 1 tasks launched. wait to complete.");
var remainingWork = new List<WorkOrder>(); var remainingWork = new List<WorkOrder>();
foreach(var lingeringTask in Phase1Tasks) foreach (var lingeringTask in Phase1Tasks)
{ {
var wo = await lingeringTask; var wo = await lingeringTask;
if(wo != null) if (wo != null)
{ {
Console.WriteLine($"articleId {wo.articleId} left a work order; it has {wo.Phase2TaskList.Count()} phase 2 task{(wo.Phase2TaskList.Count() == 1 ? "" : "s")}"); Console.WriteLine($"articleId {wo.articleId} left a work order; it has {wo.Phase2TaskList.Count()} phase 2 task{(wo.Phase2TaskList.Count() == 1 ? "" : "s")}");
remainingWork.Add(wo); remainingWork.Add(wo);
@ -97,16 +86,17 @@ namespace ttrss_co_client
#region local tasks (stripping ads, converting.) #region local tasks (stripping ads, converting.)
Console.WriteLine("===== phase 2 ====="); Console.WriteLine("===== phase 2 =====");
//loop through working directory looking for other work orders to add //loop through working directory looking for other work orders to add
var workOrderFiles = Directory.GetFiles($"{conf.WorkingDirectory}", "workorder.json", new EnumerationOptions(){RecurseSubdirectories = true}); var workOrderFiles = Directory.GetFiles($"{conf.WorkingDirectory}", "workorder.json", new EnumerationOptions() { RecurseSubdirectories = true });
Console.WriteLine($"{workOrderFiles.Count()} workorder file{(workOrderFiles.Count() == 1 ? "" : "s")} located"); Console.WriteLine($"{workOrderFiles.Count()} workorder file{(workOrderFiles.Count() == 1 ? "" : "s")} located");
foreach(var workorderFile in workOrderFiles) foreach (var workorderFile in workOrderFiles)
{ {
try try
{ {
remainingWork.Add(JsonConvert.DeserializeObject<WorkOrder>(workorderFile)); remainingWork.Add(JsonConvert.DeserializeObject<WorkOrder>(File.ReadAllText(workorderFile)));
Console.WriteLine($"picked up workorder task; {workorderFile}"); Console.WriteLine($"picked up workorder task; {workorderFile}");
File.Delete(workorderFile);
} }
catch(Exception e) catch (Exception e)
{ {
Console.Error.WriteLine($"error picking up work order file {workorderFile} - {e.Message}; {e.StackTrace}"); Console.Error.WriteLine($"error picking up work order file {workorderFile} - {e.Message}; {e.StackTrace}");
} }
@ -122,26 +112,26 @@ namespace ttrss_co_client
var phase2TaskConcretions = new List<Phase2Task>(); var phase2TaskConcretions = new List<Phase2Task>();
foreach (var phase2TaskType in phase2TaskTypes) foreach (var phase2TaskType in phase2TaskTypes)
{ {
var concretion = (Phase2Task) Activator.CreateInstance(phase2TaskType); var concretion = (Phase2Task)Activator.CreateInstance(phase2TaskType);
phase2TaskConcretions.Add(concretion); phase2TaskConcretions.Add(concretion);
} }
while(remainingWork.Count > 0) while (remainingWork.Count > 0)
{ {
//todo: solve the halting problem //todo: solve the halting problem
//ok but seriously, maybe I could time out work orders after a while? but converting a 3-hour 4k video could easily take an hour! //ok but seriously, maybe I could time out work orders after a while? but converting a 3-hour 4k video could easily take an hour!
Console.WriteLine($"{phase2TaskConcretions.Count()} phase 2 task types understood"); Console.WriteLine($"{phase2TaskConcretions.Count()} phase 2 task types understood");
var Phase2Tasks = new List<Task<Tuple<Phase2Task.TaskStatus, WorkOrder>>>(); var Phase2Tasks = new List<Task<Tuple<Phase2Task.TaskStatus, WorkOrder>>>();
Console.WriteLine($"launching first pass over work orders in phase 2."); Console.WriteLine($"launching first pass over work orders in phase 2.");
foreach(var wo in remainingWork) foreach (var wo in remainingWork)
{ {
var taskName = wo.Phase2TaskList[wo.Phase2TaskList.Keys.Min()]; var taskName = wo.Phase2TaskList[wo.Phase2TaskList.Keys.Min()];
var appropriatePhase2Task = phase2TaskConcretions.FirstOrDefault(tt => tt.TaskName == taskName); var appropriatePhase2Task = phase2TaskConcretions.FirstOrDefault(tt => tt.TaskName == taskName);
if(appropriatePhase2Task != null) if (appropriatePhase2Task != null)
{ {
wo.Phase2TaskList.Remove(wo.Phase2TaskList.Keys.Min()); wo.Phase2TaskList.Remove(wo.Phase2TaskList.Keys.Min());
Phase2Tasks.Add(appropriatePhase2Task.ActOn(wo)); Phase2Tasks.Add(appropriatePhase2Task.ActOn(wo));
@ -153,32 +143,32 @@ namespace ttrss_co_client
} }
Console.WriteLine($"phase 2 tasks launched. now the complex part."); Console.WriteLine($"phase 2 tasks launched. now the complex part.");
remainingWork = new List<WorkOrder>(); remainingWork = new List<WorkOrder>();
foreach(var lingeringTask in Phase2Tasks) foreach (var lingeringTask in Phase2Tasks)
{ {
var wo = await lingeringTask; var wo = await lingeringTask;
//if you tell me it's done, or you need to continue now.... I believe you. //if you tell me it's done, or you need to continue now.... I believe you.
//TODO: be smart enough to override? //TODO: be smart enough to override?
switch (wo.Item1) switch (wo.Item1)
{ {
case Phase2Task.TaskStatus.Done: case Phase2Task.TaskStatus.Done:
// :) Directory.Delete(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString()));
break; break;
case Phase2Task.TaskStatus.ContinueNow: case Phase2Task.TaskStatus.ContinueNow:
remainingWork.Add(wo.Item2); remainingWork.Add(wo.Item2);
break; break;
case Phase2Task.TaskStatus.TryLater: case Phase2Task.TaskStatus.TryLater:
File.WriteAllText(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString(), "workorder.json"), JsonConvert.SerializeObject(wo)); File.WriteAllText(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString(), "workorder.json"), JsonConvert.SerializeObject(wo.Item2));
break; break;
} }
} }
Console.WriteLine($"{remainingWork.Count} phase 2 tasks to be re-looped."); Console.WriteLine($"{remainingWork.Count} phase 2 tasks to be re-looped.");
} }
#endregion #endregion
await ttrssClient.Logout(); await ttrssClient.Logout();
Console.WriteLine($"logged out of ttrss."); Console.WriteLine($"logged out of ttrss.");
Console.WriteLine($"done for real"); Console.WriteLine($"done for real");
} }
static Configuration Configure(string configurationPath = "appsettings.json") static Configuration Configure(string configurationPath = "appsettings.json")

2
tasks/Convert.cs
View File

@ -17,7 +17,7 @@ namespace ttrss_co_client.tasks
} }
var sw = new Stopwatch(); var sw = new Stopwatch();
sw.Start(); sw.Start();
var conversionProc = Process.Start("ffmpeg", $"-y -i \"{wo.data["path"]}\" \"{wo.data["conversion-target"]}\""); var conversionProc = Process.Start("ffmpeg", $"-y -loglevel quiet -i \"{wo.data["path"]}\" \"{wo.data["conversion-target"]}\"");
conversionProc.WaitForExit(); conversionProc.WaitForExit();
sw.Stop(); sw.Stop();
if(File.Exists(wo.data["conversion-target"])) if(File.Exists(wo.data["conversion-target"]))

6
tasks/PodcastifyAttachment.cs
View File

@ -17,7 +17,7 @@
// // ?? hl.feed_title; // // ?? hl.feed_title;
// // var attachmentLink = hl.attachments.Select(a => a.content_url)?.FirstOrDefault(); // // var attachmentLink = hl.attachments.Select(a => a.content_url)?.FirstOrDefault();
// // var ATTpodcastifyResult = await podcastifyAttachment(attachmentLink.ToString(), podcastName, hl.title); // // var ATTpodcastifyResult = await podcastifyAttachment(attachmentLink.ToString(), podcastName, hl.title);
// // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToLongTimeString()}] - {ATTpodcastifyResult.Item2}", hl.id); // // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToString("o")}] - {ATTpodcastifyResult.Item2}", hl.id);
// // if (ATTpodcastifyResult.Item1 == true) // // if (ATTpodcastifyResult.Item1 == true)
// // { // // {
// // Console.WriteLine($" {hl.title} -> podcastify (att) success, removing labels"); // // Console.WriteLine($" {hl.title} -> podcastify (att) success, removing labels");
@ -62,7 +62,7 @@
// // Console.WriteLine($" {headline.title}: label removed"); // // Console.WriteLine($" {headline.title}: label removed");
// // if (!res.Success) // // if (!res.Success)
// // { // // {
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id); // // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id);
// // return null; // // return null;
// // } // // }
// // else // // else
@ -93,7 +93,7 @@
// // { // // {
// // Console.Error.WriteLine($"fatal error in standard DL for {headline.link}"); // // Console.Error.WriteLine($"fatal error in standard DL for {headline.link}");
// // Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}"); // // Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id); // // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id);
// // return null; // // return null;
// // } // // }

6
tasks/PodcastifyYT.cs
View File

@ -15,7 +15,7 @@
// // podcastName = nameLabel?.caption.Substring(conf.PodcastTitlePrefix.Length) // // podcastName = nameLabel?.caption.Substring(conf.PodcastTitlePrefix.Length)
// // ?? hl.feed_title; // // ?? hl.feed_title;
// // var YTpodcastifyResult = await podcastifyYT(hl.link.ToString(), podcastName); // // var YTpodcastifyResult = await podcastifyYT(hl.link.ToString(), podcastName);
// // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToLongTimeString()}] - {YTpodcastifyResult.Item2}", hl.id); // // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToString("o")}] - {YTpodcastifyResult.Item2}", hl.id);
// // if (YTpodcastifyResult.Item1 == true) // // if (YTpodcastifyResult.Item1 == true)
// // { // // {
// // Console.WriteLine($" {hl.title} -> podcastify (YT) success, removing labels"); // // Console.WriteLine($" {hl.title} -> podcastify (YT) success, removing labels");
@ -59,7 +59,7 @@
// // Console.WriteLine($" {headline.title}: label removed"); // // Console.WriteLine($" {headline.title}: label removed");
// // if (!res.Success) // // if (!res.Success)
// // { // // {
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id); // // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id);
// // return null; // // return null;
// // } // // }
// // else // // else
@ -90,7 +90,7 @@
// // { // // {
// // Console.Error.WriteLine($"fatal error in standard DL for {headline.link}"); // // Console.Error.WriteLine($"fatal error in standard DL for {headline.link}");
// // Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}"); // // Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id); // // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id);
// // return null; // // return null;
// // } // // }

19
tasks/Publish.cs
View File

@ -11,16 +11,15 @@ namespace ttrss_co_client.tasks
public override string TaskName => "filemovePublish"; public override string TaskName => "filemovePublish";
public override async Task<Tuple<TaskStatus, WorkOrder>>ActOn(WorkOrder workOrder) public override async Task<Tuple<TaskStatus, WorkOrder>>ActOn(WorkOrder workOrder)
{ {
var wo = workOrder as PublishWorkOrder; var targetDirectory = Path.GetDirectoryName(workOrder.data["publish-target"]);
File.Move(wo.Path, wo.PublishTarget, true); if(!Directory.Exists(targetDirectory))
var article = (await TtrssClient.GetArticles(wo.articleId))?.FirstOrDefault(); {
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToShortDateString()}] - copied"); Directory.CreateDirectory(targetDirectory);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.Done, wo); }
File.Move(workOrder.data["path"], workOrder.data["publish-target"], true);
var article = (await TtrssClient.GetArticles(workOrder.articleId))?.FirstOrDefault();
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToShortDateString()}] - copied", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.Done, workOrder);
} }
} }
public class PublishWorkOrder : WorkOrder
{
public string Path => data["path"];
public string PublishTarget => data["publish-target"];
}
} }

48
tasks/Sponsorblock.cs
View File

@ -1,3 +1,5 @@
using System;
using System.Text;
using Newtonsoft.Json; using Newtonsoft.Json;
using System.Linq; using System.Linq;
using System.Diagnostics; using System.Diagnostics;
@ -30,7 +32,7 @@ namespace ttrss_co_client.tasks
var videoId = match.Groups?[1].Value; var videoId = match.Groups?[1].Value;
var c = new HttpClient(); var c = new HttpClient();
var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview"); var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview&category=interaction");
IEnumerable<sponsorblock.Segment> segments = null; IEnumerable<sponsorblock.Segment> segments = null;
try try
{ {
@ -47,18 +49,19 @@ namespace ttrss_co_client.tasks
updateTimestamp = updateTimestamp.AddSeconds(article.updated).ToLocalTime(); updateTimestamp = updateTimestamp.AddSeconds(article.updated).ToLocalTime();
if (DateTime.Now - updateTimestamp > TimeSpan.FromMinutes(45)) if (DateTime.Now - updateTimestamp > TimeSpan.FromMinutes(45))
{ {
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock", article.id); await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder); return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
} }
else else
{ {
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] waiting for sponsorblock segments", article.id); await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] waiting for sponsorblock segments", article.id);
workOrder.Phase2TaskList[workOrder.Phase2TaskList.Keys.Min() - 1] = this.TaskName; workOrder.Phase2TaskList[workOrder.Phase2TaskList.Keys.Min() - 1] = this.TaskName;
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.TryLater, workOrder); return new Tuple<TaskStatus, WorkOrder>(TaskStatus.TryLater, workOrder);
} }
} }
else else
{ {
Console.WriteLine($"[{DateTime.Now.ToString("o")}] sponsorblock reports {segments.Count()} junk segments");
var contentSegments = new List<Tuple<double, double>>(); var contentSegments = new List<Tuple<double, double>>();
var previousEdge = 0.0; var previousEdge = 0.0;
var extension = workOrder.data["path"].Substring(workOrder.data["path"].LastIndexOf('.')); var extension = workOrder.data["path"].Substring(workOrder.data["path"].LastIndexOf('.'));
@ -76,37 +79,50 @@ namespace ttrss_co_client.tasks
} }
contentSegments.Add(new Tuple<double, double>(previousEdge, segments.First().videoDuration)); contentSegments.Add(new Tuple<double, double>(previousEdge, segments.First().videoDuration));
#region ffmpeg via intermediate files
var intermediateCount = 0; var intermediateCount = 0;
foreach(var seg in contentSegments) foreach(var seg in contentSegments)
{ {
var intermediateTargetPath = $"{intermediatePathBase}-intermediate-{intermediateCount.ToString("ddd")}{extension}"; var intermediateTargetPath = $"{intermediatePathBase}-intermediate-{intermediateCount.ToString("D2")}{extension}";
conversionProcesses.Add(new Tuple<Process, Tuple<int, string>>( conversionProcesses.Add(new Tuple<Process, Tuple<int, string>>(
Process.Start("ffmpeg", $"-y -i \"{workOrder.data["path"]}\" \"{intermediateTargetPath}\""), Process.Start("ffmpeg", $"-y -loglevel quiet -i \"{workOrder.data["path"]}\" -ss {seg.Item1} -to {seg.Item2} \"{intermediateTargetPath}\""),
new Tuple<int, string>(intermediateCount, intermediateTargetPath) new Tuple<int, string>(intermediateCount, intermediateTargetPath)
)); ));
Console.WriteLine("waiting for exit from task, for debugging reasons");
conversionProcesses.Last().Item1.WaitForExit();
intermediateCount++; intermediateCount++;
} }
Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments being exported"); Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments being exported");
var intermediates = new Dictionary<int, string>(); var intermediates = new Dictionary<int, string>();
foreach(var proc in conversionProcesses) foreach(var proc in conversionProcesses)
{ {
proc.Item1.WaitForExit(); proc.Item1.WaitForExit();
intermediates[proc.Item2.Item1] = proc.Item2.Item2; intermediates[proc.Item2.Item1] = proc.Item2.Item2;
} }
Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments should be exported, stitching together"); Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments should be exported, stitching together");
Process.Start("ffmpeg", $"-y -i \"concat:{string.Join('|', intermediates.OrderBy(kvp => kvp.Key).Select(kvp => kvp.Value).ToArray())}\" -c copy \"{workOrder.data["path"]}\"")
.WaitForExit();
Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments stitched. Deleting originals."); var sb = new StringBuilder();
sb.AppendLine("#ffmpeg demands it");
foreach(var intermediate in intermediates.OrderBy(kvp => kvp.Key))
{
sb.AppendLine($"file '{intermediate.Value}'");
}
var ffmpegFile = Path.Combine(Path.GetDirectoryName(workOrder.data["path"]), "ffmpeglist.txt");
File.WriteAllText(ffmpegFile, sb.ToString());
Process.Start("ffmpeg", $"-y -f concat -safe 0 -i {ffmpegFile} -c copy \"{workOrder.data["path"]}\"")
.WaitForExit();
File.Delete(ffmpegFile);
Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments stitched. Deleting originals.");
foreach(var intermediate in intermediates.Values) foreach(var intermediate in intermediates.Values)
{ {
File.Delete(intermediate); File.Delete(intermediate);
} }
#endregion
sw.Stop(); sw.Stop();
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] removed {segments.Count()} junk segments", article.id); await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] removed {segments.Count()} junk segments", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder); return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
} }

25
tasks/StandardDL.cs
View File

@ -11,13 +11,17 @@ namespace ttrss_co_client.tasks
public override async Task<WorkOrder> ActOn(Headline headline, IEnumerable<Label> labelsWRTArticle) public override async Task<WorkOrder> ActOn(Headline headline, IEnumerable<Label> labelsWRTArticle)
{ {
Console.WriteLine($" standard download: {headline.link.ToString()}"); Console.WriteLine($" standard download: {headline.link.ToString()}");
var myGuid = Guid.NewGuid(); var myGuid = Guid.NewGuid().ToString();
var ytdl = new YoutubeDLSharp.YoutubeDL(); var ytdl = new YoutubeDLSharp.YoutubeDL();
ytdl.YoutubeDLPath = "yt-dlp"; ytdl.YoutubeDLPath = "yt-dlp";
ytdl.FFmpegPath = "ffmpeg"; ytdl.FFmpegPath = "ffmpeg";
ytdl.OutputFolder = $"{Conf.WorkingDirectory}/{myGuid}"; ytdl.OutputFolder = $"{Conf.WorkingDirectory}/{myGuid}";
ytdl.OutputFileTemplate = "%(upload_date)s - %(title)s - [%(id)s].%(ext)s"; ytdl.OutputFileTemplate = "%(upload_date)s - %(title)s - [%(id)s].%(ext)s";
var sw = new Stopwatch(); var sw = new Stopwatch();
if(!Path.Exists(ytdl.OutputFolder))
{
Directory.CreateDirectory(ytdl.OutputFolder);
}
try try
{ {
@ -33,7 +37,7 @@ namespace ttrss_co_client.tasks
Console.WriteLine($" {headline.link} -> {res.Data}\n{outputStr}"); Console.WriteLine($" {headline.link} -> {res.Data}\n{outputStr}");
await TtrssClient.SetArticleLabel(labelsWRTArticle.First(l => l.caption?.ToLower() == this.TriggerLabel.ToLower()).id, false, headline.id); await TtrssClient.SetArticleLabel(labelsWRTArticle.First(l => l.caption?.ToLower() == this.TriggerLabel.ToLower()).id, false, headline.id);
Console.WriteLine($" {headline.title}: label removed"); Console.WriteLine($" {headline.title}: label removed");
await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - standard dl {(res.Success ? "success" : "failure")}; {string.Join('\n', res.ErrorOutput)}", headline.id); await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - standard dl {(res.Success ? "success" : "failure")}; {string.Join('\n', res.ErrorOutput)}", headline.id);
if (!res.Success) if (!res.Success)
{ {
return null; return null;
@ -47,20 +51,21 @@ namespace ttrss_co_client.tasks
{ {
articleId = headline.id,//<-- that way later tasks can update the note articleId = headline.id,//<-- that way later tasks can update the note
Phase2TaskList = new Dictionary<int, string>(), Phase2TaskList = new Dictionary<int, string>(),
data = new Dictionary<string, string>() data = new Dictionary<string, string>(),
guid = Guid.Parse(myGuid)
}; };
toReturn.data["path"] = outputFilename; toReturn.data["path"] = outputFilename;
if(headline.link.Host.EndsWith("youtube.com"))
{
toReturn.Phase2TaskList[0] = "sponsorblock";
}
if(!outputFilename.EndsWith(".mp4")) if(!outputFilename.EndsWith(".mp4"))
{ {
Console.WriteLine($"{headline.title} needs conversion task."); Console.WriteLine($"{headline.title} needs conversion task.");
toReturn.Phase2TaskList[1] = "convert"; toReturn.Phase2TaskList[0] = "convert";
toReturn.data["conversion-target"] = outputFilename.Substring(0, res.Data.LastIndexOf('.')) + ".mp4"; toReturn.data["conversion-target"] = outputFilename.Substring(0, res.Data.LastIndexOf('.')) + ".mp4";
} }
if(headline.link.Host.EndsWith("youtube.com"))
{
toReturn.Phase2TaskList[1] = "sponsorblock";
}
toReturn.Phase2TaskList[2] = "filemovePublish"; toReturn.Phase2TaskList[2] = "filemovePublish";
toReturn.data["publish-target"] = $"{Conf.OnDoneCopy}/recent episodes/{Path.GetFileName(toReturn.data["conversion-target"])}"; toReturn.data["publish-target"] = $"{Conf.OnDoneCopy}/recent episodes/{Path.GetFileName(toReturn.data["conversion-target"])}";
@ -71,7 +76,7 @@ namespace ttrss_co_client.tasks
{ {
Console.Error.WriteLine($"fatal error in standard DL for {headline.link}"); Console.Error.WriteLine($"fatal error in standard DL for {headline.link}");
Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}"); Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", headline.id); await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", headline.id);
return null; return null;
} }
} }