adam/ttrss-co-pilot
adam/ttrss-co-pilot
1
0
Fork 0
Code Issues 6 Pull Requests Packages Projects Releases 7 Wiki Activity

IT WORKS (again) (almost)

Browse Source 
standard DL downloads, converts first (maybe that's bad?), strips ads :)
This commit is contained in:
Adam R Grey 2023-04-09 17:01:25 -04:00
parent a64e646b04
commit 3fcc5f376d
7 changed files with 99 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Program.cs
View File

@ -11,18 +11,8 @@ namespace ttrss_co_client
private static ttrss.ApiClient ttrssClient;
static async Task Main(string[] args)
{
// var tasktypes = AppDomain.CurrentDomain.GetAssemblies()
// .SelectMany(domainAssembly => domainAssembly.GetTypes())
// .Where(type => type.IsSubclassOf(typeof(Phase1Task)) && !type.IsAbstract)
// .ToList();
// foreach (var subtype in subtypes)
// {
// var inst = (CoClientTask) Activator.CreateInstance(subtype);
// }
var conf = Configure();
Console.WriteLine($"{DateTime.Now.ToLongTimeString()}");
Console.WriteLine($"{DateTime.Now.ToString("o")}");
ttrssClient = new ttrss.ApiClient(conf.BaseURI);
await ttrssClient.Login(conf.Username, conf.Password);
@ -30,7 +20,6 @@ namespace ttrss_co_client
var loggedin = await ttrssClient.IsLoggedIn();
Console.WriteLine($"logged in: {loggedin}");
#region phase 1
Console.WriteLine("===== phase 1 =====");
@ -103,8 +92,9 @@ namespace ttrss_co_client
{
try
{
remainingWork.Add(JsonConvert.DeserializeObject<WorkOrder>(workorderFile));
remainingWork.Add(JsonConvert.DeserializeObject<WorkOrder>(File.ReadAllText(workorderFile)));
Console.WriteLine($"picked up workorder task; {workorderFile}");
File.Delete(workorderFile);
}
catch (Exception e)
{
@ -163,13 +153,13 @@ namespace ttrss_co_client
switch (wo.Item1)
{
case Phase2Task.TaskStatus.Done:
// :)
Directory.Delete(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString()));
break;
case Phase2Task.TaskStatus.ContinueNow:
remainingWork.Add(wo.Item2);
break;
case Phase2Task.TaskStatus.TryLater:
File.WriteAllText(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString(), "workorder.json"), JsonConvert.SerializeObject(wo));
File.WriteAllText(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString(), "workorder.json"), JsonConvert.SerializeObject(wo.Item2));
break;
}
}

2
tasks/Convert.cs
View File

@ -17,7 +17,7 @@ namespace ttrss_co_client.tasks
}
var sw = new Stopwatch();
sw.Start();
var conversionProc = Process.Start("ffmpeg", $"-y -i \"{wo.data["path"]}\" \"{wo.data["conversion-target"]}\"");
var conversionProc = Process.Start("ffmpeg", $"-y -loglevel quiet -i \"{wo.data["path"]}\" \"{wo.data["conversion-target"]}\"");
conversionProc.WaitForExit();
sw.Stop();
if(File.Exists(wo.data["conversion-target"]))

6
tasks/PodcastifyAttachment.cs
View File

@ -17,7 +17,7 @@
// // ?? hl.feed_title;
// // var attachmentLink = hl.attachments.Select(a => a.content_url)?.FirstOrDefault();
// // var ATTpodcastifyResult = await podcastifyAttachment(attachmentLink.ToString(), podcastName, hl.title);
// // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToLongTimeString()}] - {ATTpodcastifyResult.Item2}", hl.id);
// // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToString("o")}] - {ATTpodcastifyResult.Item2}", hl.id);
// // if (ATTpodcastifyResult.Item1 == true)
// // {
// // Console.WriteLine($" {hl.title} -> podcastify (att) success, removing labels");
@ -62,7 +62,7 @@
// // Console.WriteLine($" {headline.title}: label removed");
// // if (!res.Success)
// // {
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id);
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id);
// // return null;
// // }
// // else
@ -93,7 +93,7 @@
// // {
// // Console.Error.WriteLine($"fatal error in standard DL for {headline.link}");
// // Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id);
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id);
// // return null;
// // }

6
tasks/PodcastifyYT.cs
View File

@ -15,7 +15,7 @@
// // podcastName = nameLabel?.caption.Substring(conf.PodcastTitlePrefix.Length)
// // ?? hl.feed_title;
// // var YTpodcastifyResult = await podcastifyYT(hl.link.ToString(), podcastName);
// // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToLongTimeString()}] - {YTpodcastifyResult.Item2}", hl.id);
// // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToString("o")}] - {YTpodcastifyResult.Item2}", hl.id);
// // if (YTpodcastifyResult.Item1 == true)
// // {
// // Console.WriteLine($" {hl.title} -> podcastify (YT) success, removing labels");
@ -59,7 +59,7 @@
// // Console.WriteLine($" {headline.title}: label removed");
// // if (!res.Success)
// // {
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id);
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id);
// // return null;
// // }
// // else
@ -90,7 +90,7 @@
// // {
// // Console.Error.WriteLine($"fatal error in standard DL for {headline.link}");
// // Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id);
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id);
// // return null;
// // }

19
tasks/Publish.cs
View File

@ -11,16 +11,15 @@ namespace ttrss_co_client.tasks
public override string TaskName => "filemovePublish";
public override async Task<Tuple<TaskStatus, WorkOrder>>ActOn(WorkOrder workOrder)
{
var wo = workOrder as PublishWorkOrder;
File.Move(wo.Path, wo.PublishTarget, true);
var article = (await TtrssClient.GetArticles(wo.articleId))?.FirstOrDefault();
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToShortDateString()}] - copied");
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.Done, wo);
}
}
public class PublishWorkOrder : WorkOrder
var targetDirectory = Path.GetDirectoryName(workOrder.data["publish-target"]);
if(!Directory.Exists(targetDirectory))
{
public string Path => data["path"];
public string PublishTarget => data["publish-target"];
Directory.CreateDirectory(targetDirectory);
}
File.Move(workOrder.data["path"], workOrder.data["publish-target"], true);
var article = (await TtrssClient.GetArticles(workOrder.articleId))?.FirstOrDefault();
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToShortDateString()}] - copied", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.Done, workOrder);
}
}
}

38
tasks/Sponsorblock.cs
View File

@ -1,3 +1,5 @@
using System;
using System.Text;
using Newtonsoft.Json;
using System.Linq;
using System.Diagnostics;
@ -30,7 +32,7 @@ namespace ttrss_co_client.tasks
var videoId = match.Groups?[1].Value;
var c = new HttpClient();
var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview");
var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview&category=interaction");
IEnumerable<sponsorblock.Segment> segments = null;
try
{
@ -47,18 +49,19 @@ namespace ttrss_co_client.tasks
updateTimestamp = updateTimestamp.AddSeconds(article.updated).ToLocalTime();
if (DateTime.Now - updateTimestamp > TimeSpan.FromMinutes(45))
{
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock", article.id);
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
}
else
{
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] waiting for sponsorblock segments", article.id);
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] waiting for sponsorblock segments", article.id);
workOrder.Phase2TaskList[workOrder.Phase2TaskList.Keys.Min() - 1] = this.TaskName;
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.TryLater, workOrder);
}
}
else
{
Console.WriteLine($"[{DateTime.Now.ToString("o")}] sponsorblock reports {segments.Count()} junk segments");
var contentSegments = new List<Tuple<double, double>>();
var previousEdge = 0.0;
var extension = workOrder.data["path"].Substring(workOrder.data["path"].LastIndexOf('.'));
@ -76,37 +79,50 @@ namespace ttrss_co_client.tasks
}
contentSegments.Add(new Tuple<double, double>(previousEdge, segments.First().videoDuration));
#region ffmpeg via intermediate files
var intermediateCount = 0;
foreach(var seg in contentSegments)
{
var intermediateTargetPath = $"{intermediatePathBase}-intermediate-{intermediateCount.ToString("ddd")}{extension}";
var intermediateTargetPath = $"{intermediatePathBase}-intermediate-{intermediateCount.ToString("D2")}{extension}";
conversionProcesses.Add(new Tuple<Process, Tuple<int, string>>(
Process.Start("ffmpeg", $"-y -i \"{workOrder.data["path"]}\" \"{intermediateTargetPath}\""),
Process.Start("ffmpeg", $"-y -loglevel quiet -i \"{workOrder.data["path"]}\" -ss {seg.Item1} -to {seg.Item2} \"{intermediateTargetPath}\""),
new Tuple<int, string>(intermediateCount, intermediateTargetPath)
));
Console.WriteLine("waiting for exit from task, for debugging reasons");
conversionProcesses.Last().Item1.WaitForExit();
intermediateCount++;
}
Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments being exported");
Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments being exported");
var intermediates = new Dictionary<int, string>();
foreach(var proc in conversionProcesses)
{
proc.Item1.WaitForExit();
intermediates[proc.Item2.Item1] = proc.Item2.Item2;
}
Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments should be exported, stitching together");
Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments should be exported, stitching together");
Process.Start("ffmpeg", $"-y -i \"concat:{string.Join('|', intermediates.OrderBy(kvp => kvp.Key).Select(kvp => kvp.Value).ToArray())}\" -c copy \"{workOrder.data["path"]}\"")
var sb = new StringBuilder();
sb.AppendLine("#ffmpeg demands it");
foreach(var intermediate in intermediates.OrderBy(kvp => kvp.Key))
{
sb.AppendLine($"file '{intermediate.Value}'");
}
var ffmpegFile = Path.Combine(Path.GetDirectoryName(workOrder.data["path"]), "ffmpeglist.txt");
File.WriteAllText(ffmpegFile, sb.ToString());
Process.Start("ffmpeg", $"-y -f concat -safe 0 -i {ffmpegFile} -c copy \"{workOrder.data["path"]}\"")
.WaitForExit();
File.Delete(ffmpegFile);
Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments stitched. Deleting originals.");
Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments stitched. Deleting originals.");
foreach(var intermediate in intermediates.Values)
{
File.Delete(intermediate);
}
#endregion
sw.Stop();
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] removed {segments.Count()} junk segments", article.id);
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] removed {segments.Count()} junk segments", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
}

25
tasks/StandardDL.cs
View File

@ -11,13 +11,17 @@ namespace ttrss_co_client.tasks
public override async Task<WorkOrder> ActOn(Headline headline, IEnumerable<Label> labelsWRTArticle)
{
Console.WriteLine($" standard download: {headline.link.ToString()}");
var myGuid = Guid.NewGuid();
var myGuid = Guid.NewGuid().ToString();
var ytdl = new YoutubeDLSharp.YoutubeDL();
ytdl.YoutubeDLPath = "yt-dlp";
ytdl.FFmpegPath = "ffmpeg";
ytdl.OutputFolder = $"{Conf.WorkingDirectory}/{myGuid}";
ytdl.OutputFileTemplate = "%(upload_date)s - %(title)s - [%(id)s].%(ext)s";
var sw = new Stopwatch();
if(!Path.Exists(ytdl.OutputFolder))
{
Directory.CreateDirectory(ytdl.OutputFolder);
}
try
{
@ -33,7 +37,7 @@ namespace ttrss_co_client.tasks
Console.WriteLine($" {headline.link} -> {res.Data}\n{outputStr}");
await TtrssClient.SetArticleLabel(labelsWRTArticle.First(l => l.caption?.ToLower() == this.TriggerLabel.ToLower()).id, false, headline.id);
Console.WriteLine($" {headline.title}: label removed");
await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - standard dl {(res.Success ? "success" : "failure")}; {string.Join('\n', res.ErrorOutput)}", headline.id);
await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - standard dl {(res.Success ? "success" : "failure")}; {string.Join('\n', res.ErrorOutput)}", headline.id);
if (!res.Success)
{
return null;
@ -47,20 +51,21 @@ namespace ttrss_co_client.tasks
{
articleId = headline.id,//<-- that way later tasks can update the note
Phase2TaskList = new Dictionary<int, string>(),
data = new Dictionary<string, string>()
data = new Dictionary<string, string>(),
guid = Guid.Parse(myGuid)
};
toReturn.data["path"] = outputFilename;
if(headline.link.Host.EndsWith("youtube.com"))
{
toReturn.Phase2TaskList[0] = "sponsorblock";
}
if(!outputFilename.EndsWith(".mp4"))
{
Console.WriteLine($"{headline.title} needs conversion task.");
toReturn.Phase2TaskList[1] = "convert";
toReturn.Phase2TaskList[0] = "convert";
toReturn.data["conversion-target"] = outputFilename.Substring(0, res.Data.LastIndexOf('.')) + ".mp4";
}
if(headline.link.Host.EndsWith("youtube.com"))
{
toReturn.Phase2TaskList[1] = "sponsorblock";
}
toReturn.Phase2TaskList[2] = "filemovePublish";
toReturn.data["publish-target"] = $"{Conf.OnDoneCopy}/recent episodes/{Path.GetFileName(toReturn.data["conversion-target"])}";
@ -71,7 +76,7 @@ namespace ttrss_co_client.tasks
{
Console.Error.WriteLine($"fatal error in standard DL for {headline.link}");
Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", headline.id);
await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", headline.id);
return null;
}
}