adam/ttrss-co-pilot
adam/ttrss-co-pilot
1
0
Fork 0
Code Issues 6 Pull Requests Packages Projects Releases 7 Wiki Activity

IT WORKS (again) (almost)

Browse Source 
standard DL downloads, converts first (maybe that's bad?), strips ads :)
This commit is contained in:
Adam R Grey 2023-04-09 17:01:25 -04:00
parent a64e646b04
commit 3fcc5f376d
7 changed files with 99 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Program.cs
View File

@ -11,18 +11,8 @@ namespace ttrss_co_client
private static ttrss.ApiClient ttrssClient; private static ttrss.ApiClient ttrssClient;
static async Task Main(string[] args) static async Task Main(string[] args)
{ {
// var tasktypes = AppDomain.CurrentDomain.GetAssemblies()
// .SelectMany(domainAssembly => domainAssembly.GetTypes())
// .Where(type => type.IsSubclassOf(typeof(Phase1Task)) && !type.IsAbstract)
// .ToList();
// foreach (var subtype in subtypes)
// {
// var inst = (CoClientTask) Activator.CreateInstance(subtype);
// }
var conf = Configure(); var conf = Configure();
Console.WriteLine($"{DateTime.Now.ToLongTimeString()}"); Console.WriteLine($"{DateTime.Now.ToString("o")}");
ttrssClient = new ttrss.ApiClient(conf.BaseURI); ttrssClient = new ttrss.ApiClient(conf.BaseURI);
await ttrssClient.Login(conf.Username, conf.Password); await ttrssClient.Login(conf.Username, conf.Password);
@ -30,7 +20,6 @@ namespace ttrss_co_client
var loggedin = await ttrssClient.IsLoggedIn(); var loggedin = await ttrssClient.IsLoggedIn();
Console.WriteLine($"logged in: {loggedin}"); Console.WriteLine($"logged in: {loggedin}");
#region phase 1 #region phase 1
Console.WriteLine("===== phase 1 ====="); Console.WriteLine("===== phase 1 =====");
@ -103,8 +92,9 @@ namespace ttrss_co_client
{ {
try try
{ {
remainingWork.Add(JsonConvert.DeserializeObject<WorkOrder>(workorderFile)); remainingWork.Add(JsonConvert.DeserializeObject<WorkOrder>(File.ReadAllText(workorderFile)));
Console.WriteLine($"picked up workorder task; {workorderFile}"); Console.WriteLine($"picked up workorder task; {workorderFile}");
File.Delete(workorderFile);
} }
catch (Exception e) catch (Exception e)
{ {
@ -163,13 +153,13 @@ namespace ttrss_co_client
switch (wo.Item1) switch (wo.Item1)
{ {
case Phase2Task.TaskStatus.Done: case Phase2Task.TaskStatus.Done:
// :) Directory.Delete(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString()));
break; break;
case Phase2Task.TaskStatus.ContinueNow: case Phase2Task.TaskStatus.ContinueNow:
remainingWork.Add(wo.Item2); remainingWork.Add(wo.Item2);
break; break;
case Phase2Task.TaskStatus.TryLater: case Phase2Task.TaskStatus.TryLater:
File.WriteAllText(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString(), "workorder.json"), JsonConvert.SerializeObject(wo)); File.WriteAllText(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString(), "workorder.json"), JsonConvert.SerializeObject(wo.Item2));
break; break;
} }
} }

2
tasks/Convert.cs
View File

@ -17,7 +17,7 @@ namespace ttrss_co_client.tasks
} }
var sw = new Stopwatch(); var sw = new Stopwatch();
sw.Start(); sw.Start();
var conversionProc = Process.Start("ffmpeg", $"-y -i \"{wo.data["path"]}\" \"{wo.data["conversion-target"]}\""); var conversionProc = Process.Start("ffmpeg", $"-y -loglevel quiet -i \"{wo.data["path"]}\" \"{wo.data["conversion-target"]}\"");
conversionProc.WaitForExit(); conversionProc.WaitForExit();
sw.Stop(); sw.Stop();
if(File.Exists(wo.data["conversion-target"])) if(File.Exists(wo.data["conversion-target"]))

6
tasks/PodcastifyAttachment.cs
View File

@ -17,7 +17,7 @@
// // ?? hl.feed_title; // // ?? hl.feed_title;
// // var attachmentLink = hl.attachments.Select(a => a.content_url)?.FirstOrDefault(); // // var attachmentLink = hl.attachments.Select(a => a.content_url)?.FirstOrDefault();
// // var ATTpodcastifyResult = await podcastifyAttachment(attachmentLink.ToString(), podcastName, hl.title); // // var ATTpodcastifyResult = await podcastifyAttachment(attachmentLink.ToString(), podcastName, hl.title);
// // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToLongTimeString()}] - {ATTpodcastifyResult.Item2}", hl.id); // // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToString("o")}] - {ATTpodcastifyResult.Item2}", hl.id);
// // if (ATTpodcastifyResult.Item1 == true) // // if (ATTpodcastifyResult.Item1 == true)
// // { // // {
// // Console.WriteLine($" {hl.title} -> podcastify (att) success, removing labels"); // // Console.WriteLine($" {hl.title} -> podcastify (att) success, removing labels");
@ -62,7 +62,7 @@
// // Console.WriteLine($" {headline.title}: label removed"); // // Console.WriteLine($" {headline.title}: label removed");
// // if (!res.Success) // // if (!res.Success)
// // { // // {
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id); // // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id);
// // return null; // // return null;
// // } // // }
// // else // // else
@ -93,7 +93,7 @@
// // { // // {
// // Console.Error.WriteLine($"fatal error in standard DL for {headline.link}"); // // Console.Error.WriteLine($"fatal error in standard DL for {headline.link}");
// // Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}"); // // Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id); // // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id);
// // return null; // // return null;
// // } // // }

6
tasks/PodcastifyYT.cs
View File

@ -15,7 +15,7 @@
// // podcastName = nameLabel?.caption.Substring(conf.PodcastTitlePrefix.Length) // // podcastName = nameLabel?.caption.Substring(conf.PodcastTitlePrefix.Length)
// // ?? hl.feed_title; // // ?? hl.feed_title;
// // var YTpodcastifyResult = await podcastifyYT(hl.link.ToString(), podcastName); // // var YTpodcastifyResult = await podcastifyYT(hl.link.ToString(), podcastName);
// // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToLongTimeString()}] - {YTpodcastifyResult.Item2}", hl.id); // // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToString("o")}] - {YTpodcastifyResult.Item2}", hl.id);
// // if (YTpodcastifyResult.Item1 == true) // // if (YTpodcastifyResult.Item1 == true)
// // { // // {
// // Console.WriteLine($" {hl.title} -> podcastify (YT) success, removing labels"); // // Console.WriteLine($" {hl.title} -> podcastify (YT) success, removing labels");
@ -59,7 +59,7 @@
// // Console.WriteLine($" {headline.title}: label removed"); // // Console.WriteLine($" {headline.title}: label removed");
// // if (!res.Success) // // if (!res.Success)
// // { // // {
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id); // // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id);
// // return null; // // return null;
// // } // // }
// // else // // else
@ -90,7 +90,7 @@
// // { // // {
// // Console.Error.WriteLine($"fatal error in standard DL for {headline.link}"); // // Console.Error.WriteLine($"fatal error in standard DL for {headline.link}");
// // Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}"); // // Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id); // // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id);
// // return null; // // return null;
// // } // // }

19
tasks/Publish.cs
View File

@ -11,16 +11,15 @@ namespace ttrss_co_client.tasks
public override string TaskName => "filemovePublish"; public override string TaskName => "filemovePublish";
public override async Task<Tuple<TaskStatus, WorkOrder>>ActOn(WorkOrder workOrder) public override async Task<Tuple<TaskStatus, WorkOrder>>ActOn(WorkOrder workOrder)
{ {
var wo = workOrder as PublishWorkOrder; var targetDirectory = Path.GetDirectoryName(workOrder.data["publish-target"]);
File.Move(wo.Path, wo.PublishTarget, true); if(!Directory.Exists(targetDirectory))
var article = (await TtrssClient.GetArticles(wo.articleId))?.FirstOrDefault();
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToShortDateString()}] - copied");
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.Done, wo);
}
}
public class PublishWorkOrder : WorkOrder
{ {
public string Path => data["path"]; Directory.CreateDirectory(targetDirectory);
public string PublishTarget => data["publish-target"]; }
File.Move(workOrder.data["path"], workOrder.data["publish-target"], true);
var article = (await TtrssClient.GetArticles(workOrder.articleId))?.FirstOrDefault();
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToShortDateString()}] - copied", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.Done, workOrder);
}
} }
} }

38
tasks/Sponsorblock.cs
View File

@ -1,3 +1,5 @@
using System;
using System.Text;
using Newtonsoft.Json; using Newtonsoft.Json;
using System.Linq; using System.Linq;
using System.Diagnostics; using System.Diagnostics;
@ -30,7 +32,7 @@ namespace ttrss_co_client.tasks
var videoId = match.Groups?[1].Value; var videoId = match.Groups?[1].Value;
var c = new HttpClient(); var c = new HttpClient();
var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview"); var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview&category=interaction");
IEnumerable<sponsorblock.Segment> segments = null; IEnumerable<sponsorblock.Segment> segments = null;
try try
{ {
@ -47,18 +49,19 @@ namespace ttrss_co_client.tasks
updateTimestamp = updateTimestamp.AddSeconds(article.updated).ToLocalTime(); updateTimestamp = updateTimestamp.AddSeconds(article.updated).ToLocalTime();
if (DateTime.Now - updateTimestamp > TimeSpan.FromMinutes(45)) if (DateTime.Now - updateTimestamp > TimeSpan.FromMinutes(45))
{ {
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock", article.id); await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder); return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
} }
else else
{ {
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] waiting for sponsorblock segments", article.id); await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] waiting for sponsorblock segments", article.id);
workOrder.Phase2TaskList[workOrder.Phase2TaskList.Keys.Min() - 1] = this.TaskName; workOrder.Phase2TaskList[workOrder.Phase2TaskList.Keys.Min() - 1] = this.TaskName;
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.TryLater, workOrder); return new Tuple<TaskStatus, WorkOrder>(TaskStatus.TryLater, workOrder);
} }
} }
else else
{ {
Console.WriteLine($"[{DateTime.Now.ToString("o")}] sponsorblock reports {segments.Count()} junk segments");
var contentSegments = new List<Tuple<double, double>>(); var contentSegments = new List<Tuple<double, double>>();
var previousEdge = 0.0; var previousEdge = 0.0;
var extension = workOrder.data["path"].Substring(workOrder.data["path"].LastIndexOf('.')); var extension = workOrder.data["path"].Substring(workOrder.data["path"].LastIndexOf('.'));
@ -76,37 +79,50 @@ namespace ttrss_co_client.tasks
} }
contentSegments.Add(new Tuple<double, double>(previousEdge, segments.First().videoDuration)); contentSegments.Add(new Tuple<double, double>(previousEdge, segments.First().videoDuration));
#region ffmpeg via intermediate files
var intermediateCount = 0; var intermediateCount = 0;
foreach(var seg in contentSegments) foreach(var seg in contentSegments)
{ {
var intermediateTargetPath = $"{intermediatePathBase}-intermediate-{intermediateCount.ToString("ddd")}{extension}"; var intermediateTargetPath = $"{intermediatePathBase}-intermediate-{intermediateCount.ToString("D2")}{extension}";
conversionProcesses.Add(new Tuple<Process, Tuple<int, string>>( conversionProcesses.Add(new Tuple<Process, Tuple<int, string>>(
Process.Start("ffmpeg", $"-y -i \"{workOrder.data["path"]}\" \"{intermediateTargetPath}\""), Process.Start("ffmpeg", $"-y -loglevel quiet -i \"{workOrder.data["path"]}\" -ss {seg.Item1} -to {seg.Item2} \"{intermediateTargetPath}\""),
new Tuple<int, string>(intermediateCount, intermediateTargetPath) new Tuple<int, string>(intermediateCount, intermediateTargetPath)
)); ));
Console.WriteLine("waiting for exit from task, for debugging reasons");
conversionProcesses.Last().Item1.WaitForExit();
intermediateCount++; intermediateCount++;
} }
Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments being exported"); Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments being exported");
var intermediates = new Dictionary<int, string>(); var intermediates = new Dictionary<int, string>();
foreach(var proc in conversionProcesses) foreach(var proc in conversionProcesses)
{ {
proc.Item1.WaitForExit(); proc.Item1.WaitForExit();
intermediates[proc.Item2.Item1] = proc.Item2.Item2; intermediates[proc.Item2.Item1] = proc.Item2.Item2;
} }
Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments should be exported, stitching together"); Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments should be exported, stitching together");
Process.Start("ffmpeg", $"-y -i \"concat:{string.Join('|', intermediates.OrderBy(kvp => kvp.Key).Select(kvp => kvp.Value).ToArray())}\" -c copy \"{workOrder.data["path"]}\"") var sb = new StringBuilder();
sb.AppendLine("#ffmpeg demands it");
foreach(var intermediate in intermediates.OrderBy(kvp => kvp.Key))
{
sb.AppendLine($"file '{intermediate.Value}'");
}
var ffmpegFile = Path.Combine(Path.GetDirectoryName(workOrder.data["path"]), "ffmpeglist.txt");
File.WriteAllText(ffmpegFile, sb.ToString());
Process.Start("ffmpeg", $"-y -f concat -safe 0 -i {ffmpegFile} -c copy \"{workOrder.data["path"]}\"")
.WaitForExit(); .WaitForExit();
File.Delete(ffmpegFile);
Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments stitched. Deleting originals."); Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments stitched. Deleting originals.");
foreach(var intermediate in intermediates.Values) foreach(var intermediate in intermediates.Values)
{ {
File.Delete(intermediate); File.Delete(intermediate);
} }
#endregion
sw.Stop(); sw.Stop();
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] removed {segments.Count()} junk segments", article.id); await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] removed {segments.Count()} junk segments", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder); return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
} }

25
tasks/StandardDL.cs
View File

@ -11,13 +11,17 @@ namespace ttrss_co_client.tasks
public override async Task<WorkOrder> ActOn(Headline headline, IEnumerable<Label> labelsWRTArticle) public override async Task<WorkOrder> ActOn(Headline headline, IEnumerable<Label> labelsWRTArticle)
{ {
Console.WriteLine($" standard download: {headline.link.ToString()}"); Console.WriteLine($" standard download: {headline.link.ToString()}");
var myGuid = Guid.NewGuid(); var myGuid = Guid.NewGuid().ToString();
var ytdl = new YoutubeDLSharp.YoutubeDL(); var ytdl = new YoutubeDLSharp.YoutubeDL();
ytdl.YoutubeDLPath = "yt-dlp"; ytdl.YoutubeDLPath = "yt-dlp";
ytdl.FFmpegPath = "ffmpeg"; ytdl.FFmpegPath = "ffmpeg";
ytdl.OutputFolder = $"{Conf.WorkingDirectory}/{myGuid}"; ytdl.OutputFolder = $"{Conf.WorkingDirectory}/{myGuid}";
ytdl.OutputFileTemplate = "%(upload_date)s - %(title)s - [%(id)s].%(ext)s"; ytdl.OutputFileTemplate = "%(upload_date)s - %(title)s - [%(id)s].%(ext)s";
var sw = new Stopwatch(); var sw = new Stopwatch();
if(!Path.Exists(ytdl.OutputFolder))
{
Directory.CreateDirectory(ytdl.OutputFolder);
}
try try
{ {
@ -33,7 +37,7 @@ namespace ttrss_co_client.tasks
Console.WriteLine($" {headline.link} -> {res.Data}\n{outputStr}"); Console.WriteLine($" {headline.link} -> {res.Data}\n{outputStr}");
await TtrssClient.SetArticleLabel(labelsWRTArticle.First(l => l.caption?.ToLower() == this.TriggerLabel.ToLower()).id, false, headline.id); await TtrssClient.SetArticleLabel(labelsWRTArticle.First(l => l.caption?.ToLower() == this.TriggerLabel.ToLower()).id, false, headline.id);
Console.WriteLine($" {headline.title}: label removed"); Console.WriteLine($" {headline.title}: label removed");
await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - standard dl {(res.Success ? "success" : "failure")}; {string.Join('\n', res.ErrorOutput)}", headline.id); await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - standard dl {(res.Success ? "success" : "failure")}; {string.Join('\n', res.ErrorOutput)}", headline.id);
if (!res.Success) if (!res.Success)
{ {
return null; return null;
@ -47,20 +51,21 @@ namespace ttrss_co_client.tasks
{ {
articleId = headline.id,//<-- that way later tasks can update the note articleId = headline.id,//<-- that way later tasks can update the note
Phase2TaskList = new Dictionary<int, string>(), Phase2TaskList = new Dictionary<int, string>(),
data = new Dictionary<string, string>() data = new Dictionary<string, string>(),
guid = Guid.Parse(myGuid)
}; };
toReturn.data["path"] = outputFilename; toReturn.data["path"] = outputFilename;
if(headline.link.Host.EndsWith("youtube.com"))
{
toReturn.Phase2TaskList[0] = "sponsorblock";
}
if(!outputFilename.EndsWith(".mp4")) if(!outputFilename.EndsWith(".mp4"))
{ {
Console.WriteLine($"{headline.title} needs conversion task."); Console.WriteLine($"{headline.title} needs conversion task.");
toReturn.Phase2TaskList[1] = "convert"; toReturn.Phase2TaskList[0] = "convert";
toReturn.data["conversion-target"] = outputFilename.Substring(0, res.Data.LastIndexOf('.')) + ".mp4"; toReturn.data["conversion-target"] = outputFilename.Substring(0, res.Data.LastIndexOf('.')) + ".mp4";
} }
if(headline.link.Host.EndsWith("youtube.com"))
{
toReturn.Phase2TaskList[1] = "sponsorblock";
}
toReturn.Phase2TaskList[2] = "filemovePublish"; toReturn.Phase2TaskList[2] = "filemovePublish";
toReturn.data["publish-target"] = $"{Conf.OnDoneCopy}/recent episodes/{Path.GetFileName(toReturn.data["conversion-target"])}"; toReturn.data["publish-target"] = $"{Conf.OnDoneCopy}/recent episodes/{Path.GetFileName(toReturn.data["conversion-target"])}";
@ -71,7 +76,7 @@ namespace ttrss_co_client.tasks
{ {
Console.Error.WriteLine($"fatal error in standard DL for {headline.link}"); Console.Error.WriteLine($"fatal error in standard DL for {headline.link}");
Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}"); Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", headline.id); await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", headline.id);
return null; return null;
} }
} }