adam/ttrss-co-pilot
adam/ttrss-co-pilot
1
0
Fork 0
Code Issues 6 Pull Requests Packages Projects Releases 7 Wiki Activity

std dl, convert seems to work - adblock might, but testing is taking forever

Browse Source
This commit is contained in:
Adam R Grey 2023-04-08 02:33:38 -04:00
parent 02d6e4a0ae
commit a64e646b04
4 changed files with 121 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
Program.cs
View File

@ -287,50 +287,5 @@ namespace ttrss_co_client
return new Tuple<bool, string>(false, $"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
}
}
// private static async Task<Tuple<bool, string>> SponsorCheck(ttrss.datastructures.Headline hl)
// {
// if (!hl.link.Host.EndsWith("youtube.com"))
// {
// return new Tuple<bool, string>(true, "sponsorblock, sadly, only exists for youtube");
// }
// var match = Regex.Match(hl.link.Query, "v=([^&]+)(&|$)");
// var videoId = match.Groups?[1].Value;
// var c = new HttpClient();
// var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview");
// if (sponsorblockcheck.StatusCode == System.Net.HttpStatusCode.NotFound)
// {
// Console.WriteLine($"sponsorblock reports that {videoId} has no entries (yet)");
// var updateTimestamp = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
// updateTimestamp = updateTimestamp.AddSeconds(hl.updated).ToLocalTime();
// if (DateTime.Now - updateTimestamp > TimeSpan.FromMinutes(45))
// {
// return new Tuple<bool, string>(true, $"updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock");
// }
// else
// {
// return new Tuple<bool, string>(false, "none found, waiting");
// }
// }
// else
// {
// try
// {
// var segments = JsonConvert.DeserializeObject<IEnumerable<sponsorblock.Segment>>(await sponsorblockcheck.Content.ReadAsStringAsync());
// if (segments.Count() > 1)
// {
// return new Tuple<bool, string>(true, $"{segments.Count()} segments");
// }
// else
// {
// return new Tuple<bool, string>(false, $"no segments");
// }
// }
// catch (Exception e)
// {
// return new Tuple<bool, string>(false, $"{e.ToString()} - {e.Message}");
// }
// }
// }
}
}

3
tasks/Publish.cs
View File

@ -8,6 +8,7 @@ namespace ttrss_co_client.tasks
///<summary>Move to output</summary>
public class Publish : Phase2Task
{
public override string TaskName => "filemovePublish";
public override async Task<Tuple<TaskStatus, WorkOrder>>ActOn(WorkOrder workOrder)
{
var wo = workOrder as PublishWorkOrder;
@ -22,4 +23,4 @@ namespace ttrss_co_client.tasks
public string Path => data["path"];
public string PublishTarget => data["publish-target"];
}
}
}

126
tasks/Sponsorblock.cs
View File

@ -1,15 +1,115 @@
// using System.Diagnostics;
// using ttrss_co_client.ttrss;
// using ttrss_co_client.ttrss.datastructures;
using Newtonsoft.Json;
using System.Linq;
using System.Diagnostics;
using System.Text.RegularExpressions;
using ttrss_co_client.ttrss;
using ttrss_co_client.ttrss.datastructures;
// namespace ttrss_co_client.tasks
// {
// ///<summary>Move to output</summary>
// public class Sponsorblock : Phase2Task
// {
// public override async Task<WorkOrder> ActOn(WorkOrder wo)
// {
namespace ttrss_co_client.tasks
{
///<summary>Move to output</summary>
public class Sponsorblock : Phase2Task
{
public override string TaskName => "sponsorblock";
public override async Task<Tuple<TaskStatus, WorkOrder>>ActOn(WorkOrder workOrder)
{
var sw = new Stopwatch();
sw.Start();
var article = (await TtrssClient.GetArticles(workOrder.articleId)).FirstOrDefault();
if(article == null)
{
Console.Error.WriteLine($"couldn't find article {workOrder.articleId} for workOrder {workOrder.guid}?!");
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.Done, null);
}
if (!article.link.Host.EndsWith("youtube.com"))
{
Console.WriteLine("no sponsorblock segments on sites other than YT");
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
}
var match = Regex.Match(article.link.Query, "v=([^&]+)(&|$)");
var videoId = match.Groups?[1].Value;
var c = new HttpClient();
// }
// }
// }
var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview");
IEnumerable<sponsorblock.Segment> segments = null;
try
{
segments = JsonConvert.DeserializeObject<IEnumerable<sponsorblock.Segment>>(await sponsorblockcheck.Content.ReadAsStringAsync());
}
catch (Exception e)
{
segments = null;
}
if (sponsorblockcheck.StatusCode == System.Net.HttpStatusCode.NotFound || segments == null || segments.Count() == 0)
{
Console.WriteLine($"sponsorblock reports that {videoId} has no entries (yet)");
var updateTimestamp = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
updateTimestamp = updateTimestamp.AddSeconds(article.updated).ToLocalTime();
if (DateTime.Now - updateTimestamp > TimeSpan.FromMinutes(45))
{
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
}
else
{
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] waiting for sponsorblock segments", article.id);
workOrder.Phase2TaskList[workOrder.Phase2TaskList.Keys.Min() - 1] = this.TaskName;
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.TryLater, workOrder);
}
}
else
{
var contentSegments = new List<Tuple<double, double>>();
var previousEdge = 0.0;
var extension = workOrder.data["path"].Substring(workOrder.data["path"].LastIndexOf('.'));
var intermediatePathBase = workOrder.data["path"].Substring(0, workOrder.data["path"].LastIndexOf('.'));
var conversionProcesses = new List<Tuple<Process, Tuple<int, string>>>();
foreach(var junkSegment in segments.OrderBy(s => s.segment[0]))
{
contentSegments.Add(new Tuple<double, double>(previousEdge, junkSegment.segment[0]));
previousEdge = junkSegment.segment[1];
if(previousEdge < 0.5)
{
contentSegments.Clear();
}
}
contentSegments.Add(new Tuple<double, double>(previousEdge, segments.First().videoDuration));
var intermediateCount = 0;
foreach(var seg in contentSegments)
{
var intermediateTargetPath = $"{intermediatePathBase}-intermediate-{intermediateCount.ToString("ddd")}{extension}";
conversionProcesses.Add(new Tuple<Process, Tuple<int, string>>(
Process.Start("ffmpeg", $"-y -i \"{workOrder.data["path"]}\" \"{intermediateTargetPath}\""),
new Tuple<int, string>(intermediateCount, intermediateTargetPath)
));
intermediateCount++;
}
Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments being exported");
var intermediates = new Dictionary<int, string>();
foreach(var proc in conversionProcesses)
{
proc.Item1.WaitForExit();
intermediates[proc.Item2.Item1] = proc.Item2.Item2;
}
Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments should be exported, stitching together");
Process.Start("ffmpeg", $"-y -i \"concat:{string.Join('|', intermediates.OrderBy(kvp => kvp.Key).Select(kvp => kvp.Value).ToArray())}\" -c copy \"{workOrder.data["path"]}\"")
.WaitForExit();
Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] intermediate content segments stitched. Deleting originals.");
foreach(var intermediate in intermediates.Values)
{
File.Delete(intermediate);
}
sw.Stop();
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToLongTimeString()}] removed {segments.Count()} junk segments", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
}
}
}
}

21
tasks/StandardDL.cs
View File

@ -51,29 +51,20 @@ namespace ttrss_co_client.tasks
};
toReturn.data["path"] = outputFilename;
if(headline.link.Host.EndsWith("youtube.com"))
{
toReturn.Phase2TaskList[0] = "sponsorblock";
}
if(!outputFilename.EndsWith(".mp4"))
{
Console.WriteLine($"{headline.title} needs conversion task.");
toReturn.Phase2TaskList[0] = "convert";
toReturn.Phase2TaskList[1] = "convert";
toReturn.data["conversion-target"] = outputFilename.Substring(0, res.Data.LastIndexOf('.')) + ".mp4";
}
if(headline.link.Host.EndsWith("youtube.com"))
{
toReturn.Phase2TaskList[1] = "sponsorblock";
}
toReturn.Phase2TaskList[2] = "filemovePublish";
toReturn.data["publish-target"] = $"{Conf.OnDoneCopy}/recent episodes/{toReturn.data["conversion-target"]}";
toReturn.data["publish-target"] = $"{Conf.OnDoneCopy}/recent episodes/{Path.GetFileName(toReturn.data["conversion-target"])}";
return toReturn;
// sw.Reset();
// outputFilename =
// sw.Start();
// var conversionProc = Process.Start("ffmpeg", $"-y -i \"{res.Data}\" \"{outputFilename}\"");
// conversionProc.WaitForExit();
// sw.Stop();
// Console.WriteLine($" converted {res.Data} -> {outputFilename}");
// File.Delete(res.Data);
//outputStr += $"\nconverted in {sw.Elapsed}";
}
}
catch (Exception e)