IT WORKS (again) (almost)

standard DL downloads, converts first (maybe that's bad?), strips ads :)
This commit is contained in:
Adam R Grey 2023-04-09 17:01:25 -04:00
parent a64e646b04
commit 3fcc5f376d
7 changed files with 99 additions and 89 deletions

View File

@ -11,18 +11,8 @@ namespace ttrss_co_client
private static ttrss.ApiClient ttrssClient;
static async Task Main(string[] args)
{
// var tasktypes = AppDomain.CurrentDomain.GetAssemblies()
// .SelectMany(domainAssembly => domainAssembly.GetTypes())
// .Where(type => type.IsSubclassOf(typeof(Phase1Task)) && !type.IsAbstract)
// .ToList();
// foreach (var subtype in subtypes)
// {
// var inst = (CoClientTask) Activator.CreateInstance(subtype);
// }
var conf = Configure();
Console.WriteLine($"{DateTime.Now.ToLongTimeString()}");
Console.WriteLine($"{DateTime.Now.ToString("o")}");
ttrssClient = new ttrss.ApiClient(conf.BaseURI);
await ttrssClient.Login(conf.Username, conf.Password);
@ -30,7 +20,6 @@ namespace ttrss_co_client
var loggedin = await ttrssClient.IsLoggedIn();
Console.WriteLine($"logged in: {loggedin}");
#region phase 1
Console.WriteLine("===== phase 1 =====");
@ -45,7 +34,7 @@ namespace ttrss_co_client
var phase1TaskConcretions = new List<Phase1Task>();
foreach (var phase1TaskType in phase1TaskTypes)
{
var concretion = (Phase1Task) Activator.CreateInstance(phase1TaskType);
var concretion = (Phase1Task)Activator.CreateInstance(phase1TaskType);
concretion.TriggerLabel = conf.feedActions.FirstOrDefault(fa => fa.command == concretion.TaskName).triggerlabelCaption;
phase1TaskConcretions.Add(concretion);
}
@ -69,7 +58,7 @@ namespace ttrss_co_client
Console.WriteLine($" headline {hl.title} has label {action.triggerlabelCaption} / action {action.command}");
var appropriatePhase1Task = phase1TaskConcretions.FirstOrDefault(tt => tt.TaskName == action.command);
if(appropriatePhase1Task != null)
if (appropriatePhase1Task != null)
{
Phase1Tasks.Add(appropriatePhase1Task.ActOn(hl, labelsWRTArticle));
}
@ -82,10 +71,10 @@ namespace ttrss_co_client
}
Console.WriteLine($"done processing feeds. phase 1 tasks launched. wait to complete.");
var remainingWork = new List<WorkOrder>();
foreach(var lingeringTask in Phase1Tasks)
foreach (var lingeringTask in Phase1Tasks)
{
var wo = await lingeringTask;
if(wo != null)
if (wo != null)
{
Console.WriteLine($"articleId {wo.articleId} left a work order; it has {wo.Phase2TaskList.Count()} phase 2 task{(wo.Phase2TaskList.Count() == 1 ? "" : "s")}");
remainingWork.Add(wo);
@ -97,16 +86,17 @@ namespace ttrss_co_client
#region local tasks (stripping ads, converting.)
Console.WriteLine("===== phase 2 =====");
//loop through working directory looking for other work orders to add
var workOrderFiles = Directory.GetFiles($"{conf.WorkingDirectory}", "workorder.json", new EnumerationOptions(){RecurseSubdirectories = true});
var workOrderFiles = Directory.GetFiles($"{conf.WorkingDirectory}", "workorder.json", new EnumerationOptions() { RecurseSubdirectories = true });
Console.WriteLine($"{workOrderFiles.Count()} workorder file{(workOrderFiles.Count() == 1 ? "" : "s")} located");
foreach(var workorderFile in workOrderFiles)
foreach (var workorderFile in workOrderFiles)
{
try
{
remainingWork.Add(JsonConvert.DeserializeObject<WorkOrder>(workorderFile));
remainingWork.Add(JsonConvert.DeserializeObject<WorkOrder>(File.ReadAllText(workorderFile)));
Console.WriteLine($"picked up workorder task; {workorderFile}");
File.Delete(workorderFile);
}
catch(Exception e)
catch (Exception e)
{
Console.Error.WriteLine($"error picking up work order file {workorderFile} - {e.Message}; {e.StackTrace}");
}
@ -122,11 +112,11 @@ namespace ttrss_co_client
var phase2TaskConcretions = new List<Phase2Task>();
foreach (var phase2TaskType in phase2TaskTypes)
{
var concretion = (Phase2Task) Activator.CreateInstance(phase2TaskType);
var concretion = (Phase2Task)Activator.CreateInstance(phase2TaskType);
phase2TaskConcretions.Add(concretion);
}
while(remainingWork.Count > 0)
while (remainingWork.Count > 0)
{
//todo: solve the halting problem
//ok but seriously, maybe I could time out work orders after a while? but converting a 3-hour 4k video could easily take an hour!
@ -136,12 +126,12 @@ namespace ttrss_co_client
Console.WriteLine($"launching first pass over work orders in phase 2.");
foreach(var wo in remainingWork)
foreach (var wo in remainingWork)
{
var taskName = wo.Phase2TaskList[wo.Phase2TaskList.Keys.Min()];
var appropriatePhase2Task = phase2TaskConcretions.FirstOrDefault(tt => tt.TaskName == taskName);
if(appropriatePhase2Task != null)
if (appropriatePhase2Task != null)
{
wo.Phase2TaskList.Remove(wo.Phase2TaskList.Keys.Min());
Phase2Tasks.Add(appropriatePhase2Task.ActOn(wo));
@ -155,7 +145,7 @@ namespace ttrss_co_client
Console.WriteLine($"phase 2 tasks launched. now the complex part.");
remainingWork = new List<WorkOrder>();
foreach(var lingeringTask in Phase2Tasks)
foreach (var lingeringTask in Phase2Tasks)
{
var wo = await lingeringTask;
//if you tell me it's done, or you need to continue now.... I believe you.
@ -163,19 +153,19 @@ namespace ttrss_co_client
switch (wo.Item1)
{
case Phase2Task.TaskStatus.Done:
// :)
Directory.Delete(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString()));
break;
case Phase2Task.TaskStatus.ContinueNow:
remainingWork.Add(wo.Item2);
break;
case Phase2Task.TaskStatus.TryLater:
File.WriteAllText(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString(), "workorder.json"), JsonConvert.SerializeObject(wo));
File.WriteAllText(Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString(), "workorder.json"), JsonConvert.SerializeObject(wo.Item2));
break;
}
}
Console.WriteLine($"{remainingWork.Count} phase 2 tasks to be re-looped.");
}
#endregion
#endregion
await ttrssClient.Logout();
Console.WriteLine($"logged out of ttrss.");

View File

@ -17,7 +17,7 @@ namespace ttrss_co_client.tasks
}
var sw = new Stopwatch();
sw.Start();
var conversionProc = Process.Start("ffmpeg", $"-y -i \"{wo.data["path"]}\" \"{wo.data["conversion-target"]}\"");
var conversionProc = Process.Start("ffmpeg", $"-y -loglevel quiet -i \"{wo.data["path"]}\" \"{wo.data["conversion-target"]}\"");
conversionProc.WaitForExit();
sw.Stop();
if(File.Exists(wo.data["conversion-target"]))

View File

@ -17,7 +17,7 @@
// // ?? hl.feed_title;
// // var attachmentLink = hl.attachments.Select(a => a.content_url)?.FirstOrDefault();
// // var ATTpodcastifyResult = await podcastifyAttachment(attachmentLink.ToString(), podcastName, hl.title);
// // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToLongTimeString()}] - {ATTpodcastifyResult.Item2}", hl.id);
// // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToString("o")}] - {ATTpodcastifyResult.Item2}", hl.id);
// // if (ATTpodcastifyResult.Item1 == true)
// // {
// // Console.WriteLine($" {hl.title} -> podcastify (att) success, removing labels");
@ -62,7 +62,7 @@
// // Console.WriteLine($" {headline.title}: label removed");
// // if (!res.Success)
// // {
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id);
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id);
// // return null;
// // }
// // else
@ -93,7 +93,7 @@
// // {
// // Console.Error.WriteLine($"fatal error in standard DL for {headline.link}");
// // Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id);
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id);
// // return null;
// // }

View File

@ -15,7 +15,7 @@
// // podcastName = nameLabel?.caption.Substring(conf.PodcastTitlePrefix.Length)
// // ?? hl.feed_title;
// // var YTpodcastifyResult = await podcastifyYT(hl.link.ToString(), podcastName);
// // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToLongTimeString()}] - {YTpodcastifyResult.Item2}", hl.id);
// // await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToString("o")}] - {YTpodcastifyResult.Item2}", hl.id);
// // if (YTpodcastifyResult.Item1 == true)
// // {
// // Console.WriteLine($" {hl.title} -> podcastify (YT) success, removing labels");
@ -59,7 +59,7 @@
// // Console.WriteLine($" {headline.title}: label removed");
// // if (!res.Success)
// // {
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id);
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - standard dl failed; {string.Join('\n', res.ErrorOutput)}", hl.id);
// // return null;
// // }
// // else
@ -90,7 +90,7 @@
// // {
// // Console.Error.WriteLine($"fatal error in standard DL for {headline.link}");
// // Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id);
// // await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", hl.id);
// // return null;
// // }

View File

@ -11,16 +11,15 @@ namespace ttrss_co_client.tasks
public override string TaskName => "filemovePublish";
public override async Task<Tuple<TaskStatus, WorkOrder>>ActOn(WorkOrder workOrder)
{
var wo = workOrder as PublishWorkOrder;
File.Move(wo.Path, wo.PublishTarget, true);
var article = (await TtrssClient.GetArticles(wo.articleId))?.FirstOrDefault();
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToShortDateString()}] - copied");
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.Done, wo);
}
}
public class PublishWorkOrder : WorkOrder
var targetDirectory = Path.GetDirectoryName(workOrder.data["publish-target"]);
if(!Directory.Exists(targetDirectory))
{
public string Path => data["path"];
public string PublishTarget => data["publish-target"];
Directory.CreateDirectory(targetDirectory);
}
File.Move(workOrder.data["path"], workOrder.data["publish-target"], true);
var article = (await TtrssClient.GetArticles(workOrder.articleId))?.FirstOrDefault();
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToShortDateString()}] - copied", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.Done, workOrder);
}
}
}

View File

@ -1,3 +1,5 @@
using System;
using System.Text;
using Newtonsoft.Json;
using System.Linq;
using System.Diagnostics;
@ -30,7 +32,7 @@ namespace ttrss_co_client.tasks
var videoId = match.Groups?[1].Value;
var c = new HttpClient();
var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview");
var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview&category=interaction");
IEnumerable<sponsorblock.Segment> segments = null;
try
{
@ -47,18 +49,19 @@ namespace ttrss_co_client.tasks
updateTimestamp = updateTimestamp.AddSeconds(article.updated).ToLocalTime();
if (DateTime.Now - updateTimestamp > TimeSpan.FromMinutes(45))
{
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock", article.id);
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
}
else
{
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] waiting for sponsorblock segments", article.id);
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] waiting for sponsorblock segments", article.id);
workOrder.Phase2TaskList[workOrder.Phase2TaskList.Keys.Min() - 1] = this.TaskName;
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.TryLater, workOrder);
}
}
else
{
Console.WriteLine($"[{DateTime.Now.ToString("o")}] sponsorblock reports {segments.Count()} junk segments");
var contentSegments = new List<Tuple<double, double>>();
var previousEdge = 0.0;
var extension = workOrder.data["path"].Substring(workOrder.data["path"].LastIndexOf('.'));
@ -76,37 +79,50 @@ namespace ttrss_co_client.tasks
}
contentSegments.Add(new Tuple<double, double>(previousEdge, segments.First().videoDuration));
#region ffmpeg via intermediate files
var intermediateCount = 0;
foreach(var seg in contentSegments)
{
var intermediateTargetPath = $"{intermediatePathBase}-intermediate-{intermediateCount.ToString("ddd")}{extension}";
var intermediateTargetPath = $"{intermediatePathBase}-intermediate-{intermediateCount.ToString("D2")}{extension}";
conversionProcesses.Add(new Tuple<Process, Tuple<int, string>>(
Process.Start("ffmpeg", $"-y -i \"{workOrder.data["path"]}\" \"{intermediateTargetPath}\""),
Process.Start("ffmpeg", $"-y -loglevel quiet -i \"{workOrder.data["path"]}\" -ss {seg.Item1} -to {seg.Item2} \"{intermediateTargetPath}\""),
new Tuple<int, string>(intermediateCount, intermediateTargetPath)
));
Console.WriteLine("waiting for exit from task, for debugging reasons");
conversionProcesses.Last().Item1.WaitForExit();
intermediateCount++;
}
Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments being exported");
Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments being exported");
var intermediates = new Dictionary<int, string>();
foreach(var proc in conversionProcesses)
{
proc.Item1.WaitForExit();
intermediates[proc.Item2.Item1] = proc.Item2.Item2;
}
Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments should be exported, stitching together");
Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments should be exported, stitching together");
Process.Start("ffmpeg", $"-y -i \"concat:{string.Join('|', intermediates.OrderBy(kvp => kvp.Key).Select(kvp => kvp.Value).ToArray())}\" -c copy \"{workOrder.data["path"]}\"")
var sb = new StringBuilder();
sb.AppendLine("#ffmpeg demands it");
foreach(var intermediate in intermediates.OrderBy(kvp => kvp.Key))
{
sb.AppendLine($"file '{intermediate.Value}'");
}
var ffmpegFile = Path.Combine(Path.GetDirectoryName(workOrder.data["path"]), "ffmpeglist.txt");
File.WriteAllText(ffmpegFile, sb.ToString());
Process.Start("ffmpeg", $"-y -f concat -safe 0 -i {ffmpegFile} -c copy \"{workOrder.data["path"]}\"")
.WaitForExit();
File.Delete(ffmpegFile);
Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments stitched. Deleting originals.");
Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments stitched. Deleting originals.");
foreach(var intermediate in intermediates.Values)
{
File.Delete(intermediate);
}
#endregion
sw.Stop();
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] removed {segments.Count()} junk segments", article.id);
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] removed {segments.Count()} junk segments", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
}

View File

@ -11,13 +11,17 @@ namespace ttrss_co_client.tasks
public override async Task<WorkOrder> ActOn(Headline headline, IEnumerable<Label> labelsWRTArticle)
{
Console.WriteLine($" standard download: {headline.link.ToString()}");
var myGuid = Guid.NewGuid();
var myGuid = Guid.NewGuid().ToString();
var ytdl = new YoutubeDLSharp.YoutubeDL();
ytdl.YoutubeDLPath = "yt-dlp";
ytdl.FFmpegPath = "ffmpeg";
ytdl.OutputFolder = $"{Conf.WorkingDirectory}/{myGuid}";
ytdl.OutputFileTemplate = "%(upload_date)s - %(title)s - [%(id)s].%(ext)s";
var sw = new Stopwatch();
if(!Path.Exists(ytdl.OutputFolder))
{
Directory.CreateDirectory(ytdl.OutputFolder);
}
try
{
@ -33,7 +37,7 @@ namespace ttrss_co_client.tasks
Console.WriteLine($" {headline.link} -> {res.Data}\n{outputStr}");
await TtrssClient.SetArticleLabel(labelsWRTArticle.First(l => l.caption?.ToLower() == this.TriggerLabel.ToLower()).id, false, headline.id);
Console.WriteLine($" {headline.title}: label removed");
await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - standard dl {(res.Success ? "success" : "failure")}; {string.Join('\n', res.ErrorOutput)}", headline.id);
await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - standard dl {(res.Success ? "success" : "failure")}; {string.Join('\n', res.ErrorOutput)}", headline.id);
if (!res.Success)
{
return null;
@ -47,20 +51,21 @@ namespace ttrss_co_client.tasks
{
articleId = headline.id,//<-- that way later tasks can update the note
Phase2TaskList = new Dictionary<int, string>(),
data = new Dictionary<string, string>()
data = new Dictionary<string, string>(),
guid = Guid.Parse(myGuid)
};
toReturn.data["path"] = outputFilename;
if(headline.link.Host.EndsWith("youtube.com"))
{
toReturn.Phase2TaskList[0] = "sponsorblock";
}
if(!outputFilename.EndsWith(".mp4"))
{
Console.WriteLine($"{headline.title} needs conversion task.");
toReturn.Phase2TaskList[1] = "convert";
toReturn.Phase2TaskList[0] = "convert";
toReturn.data["conversion-target"] = outputFilename.Substring(0, res.Data.LastIndexOf('.')) + ".mp4";
}
if(headline.link.Host.EndsWith("youtube.com"))
{
toReturn.Phase2TaskList[1] = "sponsorblock";
}
toReturn.Phase2TaskList[2] = "filemovePublish";
toReturn.data["publish-target"] = $"{Conf.OnDoneCopy}/recent episodes/{Path.GetFileName(toReturn.data["conversion-target"])}";
@ -71,7 +76,7 @@ namespace ttrss_co_client.tasks
{
Console.Error.WriteLine($"fatal error in standard DL for {headline.link}");
Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToLongTimeString()}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", headline.id);
await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", headline.id);
return null;
}
}