Compare commits

...

19 Commits

Author SHA1 Message Date
54d2afac8d attempt to clean up working directories 2023-11-12 16:23:38 -05:00
983a57b3d7 configurable user agent 2023-11-12 16:08:13 -05:00
c30deeed49 workaround to be allowed to download podcasts automatically
you know, like a... like a um.. what do you call those...

podcast
2023-11-12 15:09:15 -05:00
2b0586b0ac tell me the title 2023-11-12 15:00:01 -05:00
a832635a21 back to not always writing workorder file 2023-07-17 16:22:26 -04:00
5309609bfd mark as read and chat a message when done 2023-07-06 22:31:53 -04:00
9d252ee0d3 on publish, another pass to filter distasteful chars in filenames
turns out *nix has _no_ problem with colons, but android does
2023-04-15 08:25:38 -04:00
7e6b0fbc40 fix bug with podcastify attachment
it was making a file, same name as destination.
This is why we oop!
2023-04-10 23:20:07 -04:00
3d84beb640 yt-dlp process extracted 2023-04-10 15:25:47 -04:00
3958eee080 filter out distasteful characters. namely apostrophe.
not an illegal character in a file path, which yt-dlp would handle for me, but given how I send it off to ffmpeg I trip over this sometimes
2023-04-10 14:43:27 -04:00
036dff3b12 podcast from attachment works 2023-04-09 18:00:28 -04:00
3b11384708 podcastify YT works 2023-04-09 17:26:20 -04:00
3fcc5f376d IT WORKS (again) (almost)
standard DL downloads, converts first (maybe that's bad?), strips ads :)
2023-04-09 17:01:25 -04:00
a64e646b04 std dl, convert seems to work - adblock might, but testing is taking forever 2023-04-08 02:33:38 -04:00
02d6e4a0ae convert seems to work 2023-04-07 23:54:09 -04:00
826b181cf1 a whole new method - work order mementos 2023-04-07 22:24:31 -04:00
9628a7b8b9 oh wait, i didn't believe it when it said it was broken 2023-04-06 16:01:26 -04:00
3816c29612 thoroughly check sponsorblock segments 2023-04-06 16:00:15 -04:00
2baaa4b1ef report sponsor check status 2023-04-06 14:17:31 -04:00
17 changed files with 739 additions and 261 deletions

2
.gitignore vendored
View File

@ -1,4 +1,4 @@
tmp/ working/
# ---> VisualStudio # ---> VisualStudio
## Ignore Visual Studio temporary files, build results, and ## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons. ## files generated by popular Visual Studio add-ons.

View File

@ -9,6 +9,9 @@ namespace ttrss_co_client
public string Password { get; set; } public string Password { get; set; }
public string PodcastTitlePrefix { get; set; } public string PodcastTitlePrefix { get; set; }
public string OnDoneCopy { get; set; } public string OnDoneCopy { get; set; }
public string WorkingDirectory { get; set; } = "./working/";
public string ChatScript { get; set; }
public string UserAgent { get; set; }
public IEnumerable<FeedAction> feedActions { get; set; } public IEnumerable<FeedAction> feedActions { get; set; }
public class FeedAction public class FeedAction
{ {

View File

@ -2,148 +2,202 @@
using System.Linq; using System.Linq;
using System.Diagnostics; using System.Diagnostics;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using ttrss_co_client.tasks;
namespace ttrss_co_client namespace ttrss_co_client
{ {
class Program class Program
{ {
private static ttrss.ApiClient ttrssClient;
static async Task Main(string[] args) static async Task Main(string[] args)
{ {
var conf = Configure(); var conf = Configure();
Console.WriteLine($"{DateTime.Now.ToLongTimeString()}"); Console.WriteLine($"{DateTime.Now.ToString("o")}");
var ttrssClient = new ttrss.ApiClient(conf.BaseURI); ttrssClient = new ttrss.ApiClient(conf.BaseURI);
await ttrssClient.Login(conf.Username, conf.Password); await ttrssClient.Login(conf.Username, conf.Password);
var loggedin = await ttrssClient.IsLoggedIn(); var loggedin = await ttrssClient.IsLoggedIn();
Console.WriteLine($"logged in: {loggedin}"); Console.WriteLine($"logged in: {loggedin}");
#region phase 1
Console.WriteLine("===== phase 1 =====");
var Phase1Tasks = new List<Task<WorkOrder>>();
Phase1Task.TtrssClient = ttrssClient;
Phase1Task.Conf = conf;
var phase1TaskTypes = AppDomain.CurrentDomain.GetAssemblies()
.SelectMany(domainAssembly => domainAssembly.GetTypes())
.Where(type => type.IsSubclassOf(typeof(Phase1Task)) && !type.IsAbstract)
.ToList();
var phase1TaskConcretions = new List<Phase1Task>();
foreach (var phase1TaskType in phase1TaskTypes)
{
var concretion = (Phase1Task)Activator.CreateInstance(phase1TaskType);
concretion.TriggerLabel = conf.feedActions.FirstOrDefault(fa => fa.command == concretion.TaskName).triggerlabelCaption;
phase1TaskConcretions.Add(concretion);
}
Console.WriteLine($"{phase1TaskConcretions.Count()} phase 1 task types understood");
var unreadFeeds = await ttrssClient.GetFeeds(cat_id: -3, unread_only: true); var unreadFeeds = await ttrssClient.GetFeeds(cat_id: -3, unread_only: true);
Console.WriteLine($"{unreadFeeds.Count()} feeds unread");
foreach (var uf in unreadFeeds) foreach (var uf in unreadFeeds)
{ {
Console.WriteLine($"unread feed: {uf.title}"); Console.WriteLine($"unread feed: {uf.title}");
var headlines = await ttrssClient.GetHeadlines(uf.id, view_mode: ttrss.ApiClient.VIEWMODE.Unread, include_attachments: true); var headlines = await ttrssClient.GetHeadlines(uf.id, view_mode: ttrss.ApiClient.VIEWMODE.Unread, include_attachments: true);
foreach (var hl in headlines) foreach (var hl in headlines)
{ {
var labelsWRTFeed = (await ttrssClient.GetLabels(hl.id)); var labelsWRTArticle = (await ttrssClient.GetLabels(hl.id));
var actionsForFeed = conf.feedActions.Where(fa => var actionsForFeed = conf.feedActions.Where(fa =>
labelsWRTFeed.Where(l => l.@checked).Select(l => l.caption).Contains(fa.triggerlabelCaption))?.ToList(); labelsWRTArticle.Where(l => l.@checked).Select(l => l.caption).Contains(fa.triggerlabelCaption))?.ToList();
if (actionsForFeed != null && actionsForFeed.Any()) if (actionsForFeed != null && actionsForFeed.Any())
{ {
if(!(await SponsorCheck(hl))) var action = actionsForFeed.First();
Console.WriteLine($" headline {hl.title} has label {action.triggerlabelCaption} / action {action.command}");
var appropriatePhase1Task = phase1TaskConcretions.FirstOrDefault(tt => tt.TaskName == action.command);
if (appropriatePhase1Task != null)
{ {
await ttrssClient.UpdateArticleNote($"{hl.note}\n[{DateTime.Now.ToLongTimeString()}] waiting for sponsorblock", hl.id); Phase1Tasks.Add(appropriatePhase1Task.ActOn(hl, labelsWRTArticle));
continue;
} }
foreach (var action in actionsForFeed) else
{ {
Console.WriteLine($" {hl.title} -> action: {action.command}"); Console.Error.WriteLine($"couldn't find phase 1 task {action.command} for workorder referring to article id {hl.id}!");
var noteString = hl.note;
ttrss.datastructures.Label nameLabel;
string podcastName;
if (!string.IsNullOrWhiteSpace(noteString))
{
noteString += $"{hl.note}\n";
}
switch (action.command)
{
case "dl":
var stdDLResult = await standardDL(hl.link.ToString());
await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToLongTimeString()}] - {stdDLResult.Item2}", hl.id);
if (stdDLResult.Item1 == true)
{
Console.WriteLine($" {hl.title} -> dl success, removing label");
await ttrssClient.SetArticleLabel(
labelsWRTFeed.First(l => l.caption == action.triggerlabelCaption).id,
false,
hl.id);
}
else
{
Console.WriteLine($" {hl.title} -> dl failed");
}
break;
case "podcastifyYT":
nameLabel = labelsWRTFeed.FirstOrDefault(l => l.caption?.StartsWith(conf.PodcastTitlePrefix) == true && l.@checked);
podcastName = nameLabel?.caption.Substring(conf.PodcastTitlePrefix.Length)
?? hl.feed_title;
var YTpodcastifyResult = await podcastifyYT(hl.link.ToString(), podcastName);
await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToLongTimeString()}] - {YTpodcastifyResult.Item2}", hl.id);
if (YTpodcastifyResult.Item1 == true)
{
Console.WriteLine($" {hl.title} -> podcastify (YT) success, removing labels");
await ttrssClient.SetArticleLabel(
labelsWRTFeed.First(l => l.caption == action.triggerlabelCaption).id,
false,
hl.id);
if (nameLabel != null)
{
await ttrssClient.SetArticleLabel(nameLabel.id, false, hl.id);
}
}
else
{
Console.WriteLine($" {hl.title} -> podcastify (YT) failed");
}
break;
case "podcastifyAttachment":
nameLabel = labelsWRTFeed.FirstOrDefault(l => l.caption?.StartsWith(conf.PodcastTitlePrefix) == true && l.@checked);
podcastName = nameLabel?.caption.Substring(conf.PodcastTitlePrefix.Length)
?? hl.feed_title;
var attachmentLink = hl.attachments.Select(a => a.content_url)?.FirstOrDefault();
var ATTpodcastifyResult = await podcastifyAttachment(attachmentLink.ToString(), podcastName, hl.title);
await ttrssClient.UpdateArticleNote($"{noteString}[{DateTime.Now.ToLongTimeString()}] - {ATTpodcastifyResult.Item2}", hl.id);
if (ATTpodcastifyResult.Item1 == true)
{
Console.WriteLine($" {hl.title} -> podcastify (att) success, removing labels");
await ttrssClient.SetArticleLabel(
labelsWRTFeed.First(l => l.caption == action.triggerlabelCaption).id,
false,
hl.id);
if (nameLabel != null)
{
await ttrssClient.SetArticleLabel(nameLabel.id, false, hl.id);
}
}
else
{
Console.WriteLine($" {hl.title} -> podcastify (att) failed");
}
break;
default:
noteString += $"[{DateTime.Now.ToLongTimeString()}] - feed configured but action not recognized";
break;
}
} }
} }
} }
} }
Console.WriteLine($"done processing feeds. phase 1 tasks launched. wait to complete.");
var remainingWork = new List<WorkOrder>();
Console.WriteLine($"logging out"); foreach (var lingeringTask in Phase1Tasks)
await ttrssClient.Logout();
Console.WriteLine($"done, moving files from temporary location");
var resulted = Directory.GetFiles("tmp", "*.*", SearchOption.AllDirectories);
if (resulted.Count() > 0)
{ {
foreach (var f in resulted) var wo = await lingeringTask;
if (wo != null)
{ {
var moveTarget = Path.Combine(conf.OnDoneCopy, f.Substring("tmp/".Length)); Console.WriteLine($"articleId {wo.articleId} left a work order; it has {wo.Phase2TaskList.Count()} phase 2 task{(wo.Phase2TaskList.Count() == 1 ? "" : "s")}");
if (!Path.Exists(Path.GetDirectoryName(moveTarget))) remainingWork.Add(wo);
}
}
Console.WriteLine($"phase 1 tasks complete, carrying {remainingWork.Count()} workorder{(remainingWork.Count() == 1 ? "" : "s")} to phase 2");
#endregion
#region local tasks (stripping ads, converting.)
Console.WriteLine("===== phase 2 =====");
//loop through working directory looking for other work orders to add
var workOrderFiles = Directory.GetFiles($"{conf.WorkingDirectory}", "workorder.json", new EnumerationOptions() { RecurseSubdirectories = true });
Console.WriteLine($"{workOrderFiles.Count()} workorder file{(workOrderFiles.Count() == 1 ? "" : "s")} located");
foreach (var workorderFile in workOrderFiles)
{
try
{
remainingWork.Add(JsonConvert.DeserializeObject<WorkOrder>(File.ReadAllText(workorderFile)));
Console.WriteLine($"picked up workorder task; {workorderFile}");
File.Delete(workorderFile);
}
catch (Exception e)
{
Console.Error.WriteLine($"error picking up work order file {workorderFile} - {e.Message}; {e.StackTrace}");
}
}
Console.WriteLine($"{remainingWork.Count} phase 2 workorders, between pulled from ttrss and read from files");
Phase2Task.TtrssClient = ttrssClient;
Phase2Task.Conf = conf;
var phase2TaskTypes = AppDomain.CurrentDomain.GetAssemblies()
.SelectMany(domainAssembly => domainAssembly.GetTypes())
.Where(type => type.IsSubclassOf(typeof(Phase2Task)) && !type.IsAbstract)
.ToList();
var phase2TaskConcretions = new List<Phase2Task>();
foreach (var phase2TaskType in phase2TaskTypes)
{
var concretion = (Phase2Task)Activator.CreateInstance(phase2TaskType);
phase2TaskConcretions.Add(concretion);
}
ChatMessage.ChatScript = conf.ChatScript;
while (remainingWork.Count > 0)
{
//todo: solve the halting problem
//ok but seriously, maybe I could time out work orders after a while? but converting a 3-hour 4k video could easily take an hour!
Console.WriteLine($"{phase2TaskConcretions.Count()} phase 2 task types understood");
var Phase2Tasks = new List<Task<Tuple<Phase2Task.TaskStatus, WorkOrder>>>();
Console.WriteLine($"launching first pass over work orders in phase 2.");
foreach (var wo in remainingWork)
{
var taskName = wo.Phase2TaskList[wo.Phase2TaskList.Keys.Min()];
var appropriatePhase2Task = phase2TaskConcretions.FirstOrDefault(tt => tt.TaskName == taskName);
if (appropriatePhase2Task != null)
{ {
Directory.CreateDirectory(Path.GetDirectoryName(moveTarget)); wo.Phase2TaskList.Remove(wo.Phase2TaskList.Keys.Min());
} Console.WriteLine("launching phase 2 task: " + taskName);
if(File.Exists(moveTarget)) Phase2Tasks.Add(appropriatePhase2Task.ActOn(wo));
{
Console.WriteLine("file already exists, abandoning");
} }
else else
{ {
File.Move(f, moveTarget); Console.Error.WriteLine($"couldn't find phase 2 task {taskName} for workorder referring to article id {wo.articleId}!");
} }
} }
Console.WriteLine($"phase 2 tasks launched. now the complex part.");
remainingWork = new List<WorkOrder>();
foreach (var lingeringTask in Phase2Tasks)
{
var wo = await lingeringTask;
var wop = Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString(), "workorder.json");
//if you tell me it's done, or you need to continue now.... I believe you.
switch (wo.Item1)
{
case Phase2Task.TaskStatus.Done:
var workingSubDir = Path.Combine(conf.WorkingDirectory, wo.Item2.guid.ToString());
foreach(var file in Directory.GetFiles(workingSubDir, "*.*", SearchOption.AllDirectories))
{
File.Delete(file);
}
//imo this next line should also handle the above, but apparently I'm alone in that.
Directory.Delete(workingSubDir, true);
break;
case Phase2Task.TaskStatus.ContinueNow:
remainingWork.Add(wo.Item2);
break;
case Phase2Task.TaskStatus.TryLater:
File.WriteAllText(wop, JsonConvert.SerializeObject(wo.Item2));
break;
}
}
Console.WriteLine($"{remainingWork.Count} phase 2 tasks to be re-looped.");
} }
#endregion
await ttrssClient.Logout();
Console.WriteLine($"logged out of ttrss.");
#region cleanup
Console.WriteLine("===== phase 3 =====");
//loop through working directory looking for other work orders to add
var subdirs = Directory.GetDirectories(conf.WorkingDirectory);
foreach(var subdir in subdirs)
{
if(Directory.GetFiles(subdir)?.Length == 0)
{
Console.WriteLine($"{subdir} is empty");
Directory.Delete(subdir);
}
else
{
Console.WriteLine($"{subdir} is not empty: {string.Join(", ", Directory.GetFiles(subdir))}");
}
}
#endregion
Console.WriteLine($"done for real"); Console.WriteLine($"done for real");
} }
static Configuration Configure(string configurationPath = "appsettings.json") static Configuration Configure(string configurationPath = "appsettings.json")
@ -173,160 +227,5 @@ namespace ttrss_co_client
return conf; return conf;
} }
private static async Task<Tuple<bool, string>> standardDL(string articleLink)
{
Console.WriteLine($" standard downloading {articleLink}");
try
{
var ytdl = new YoutubeDLSharp.YoutubeDL();
ytdl.YoutubeDLPath = "yt-dlp";
ytdl.FFmpegPath = "ffmpeg";
ytdl.OutputFolder = "./tmp/recent episodes";
ytdl.OutputFileTemplate = "%(upload_date)s - %(title)s - [%(id)s].%(ext)s";
var sw = new Stopwatch();
sw.Start();
var res = await ytdl.RunVideoDownload(articleLink);
sw.Stop();
var outputStr = $"{(res.Success ? "Success" : "fail")} in {sw.Elapsed}";
if (res.ErrorOutput != null && res.ErrorOutput.Length > 0)
{
outputStr += "\n" + string.Join('\n', res.ErrorOutput);
}
Console.WriteLine($" download {(res.Success ? "success" : "failed")}: {articleLink} -> {res.Data}");
if (!res.Data.EndsWith(".mp4"))
{
Console.WriteLine(" must convert video");
sw.Reset();
var outputFilename = res.Data.Substring(0, res.Data.LastIndexOf('.')) + ".mp4";
sw.Start();
var conversionProc = Process.Start("ffmpeg", $"-y -i \"{res.Data}\" \"{outputFilename}\"");
conversionProc.WaitForExit();
sw.Stop();
Console.WriteLine($" converted {res.Data} -> {outputFilename}");
File.Delete(res.Data);
outputStr += $"\nconverted in {sw.Elapsed}";
}
return new Tuple<bool, string>(res.Success, outputStr);
}
catch (Exception e)
{
Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
return new Tuple<bool, string>(false, $"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
}
}
private static async Task<Tuple<bool, string>> podcastifyYT(string articleLink, string podcastName)
{
Console.WriteLine($" youtube-podcastifying {articleLink} ({podcastName})");
try
{
var ytdl = new YoutubeDLSharp.YoutubeDL();
ytdl.YoutubeDLPath = "yt-dlp";
ytdl.FFmpegPath = "ffmpeg";
ytdl.OutputFolder = $"./tmp/Podcasts/{podcastName}";
ytdl.OutputFileTemplate = "%(upload_date)s - %(title)s - [%(id)s].%(ext)s";
var sw = new Stopwatch();
sw.Start();
var res = await ytdl.RunAudioDownload(articleLink);
sw.Stop();
var outputStr = $"{(res.Success ? "Success" : "fail")} in {sw.Elapsed}";
if (res.ErrorOutput != null && res.ErrorOutput.Length > 0)
{
outputStr += "\n" + string.Join('\n', res.ErrorOutput);
}
Console.WriteLine($" {(res.Success ? "Success" : "fail")} in {sw.Elapsed} - {res.Data}");
if (res.Success && !res.Data.EndsWith(".mp3"))
{
Console.WriteLine(" must convert audio");
sw.Reset();
var outputFilename = res.Data.Substring(0, res.Data.LastIndexOf('.')) + ".mp3";
sw.Start();
var conversionProc = Process.Start("ffmpeg", $"-y -i \"{res.Data}\" \"{outputFilename}\"");
conversionProc.WaitForExit();
sw.Stop();
File.Delete(res.Data);
outputStr += $"\nconverted in {sw.Elapsed}";
}
return new Tuple<bool, string>(res.Success, outputStr);
}
catch (Exception e)
{
Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
return new Tuple<bool, string>(false, $"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
}
}
private static async Task<Tuple<bool, string>> podcastifyAttachment(string attachmentLink, string podcastName, string episodeTitle)
{
Console.WriteLine($" attachment-podcastifying {attachmentLink} ({podcastName})");
try
{
var extensionUpstream = attachmentLink.Substring(attachmentLink.LastIndexOf('.'));
var containingDirectory = $"./tmp/Podcasts/{podcastName}/";
var outputFilename = $"{containingDirectory}{episodeTitle}{extensionUpstream}";
if(!Directory.Exists(containingDirectory))
{
Directory.CreateDirectory(containingDirectory);
}
var downloader = new HttpClient();
var sw = new Stopwatch();
sw.Start();
File.WriteAllBytes(outputFilename, await (await downloader.GetAsync(attachmentLink)).Content.ReadAsByteArrayAsync());
sw.Stop();
var outputStr = $"{(File.Exists(outputFilename) ? "Success" : "fail")} in {sw.Elapsed}";
Console.WriteLine($" {outputStr} - {outputFilename}");
if (File.Exists(outputFilename) && extensionUpstream != ".mp3")
{
Console.WriteLine(" must convert audio");
sw.Reset();
var targetFilename = outputFilename.Substring(0, outputFilename.LastIndexOf('.')) + ".mp3";
sw.Start();
var conversionProc = Process.Start("ffmpeg", $"-y -i \"{outputFilename}\" \"{targetFilename}\"");
conversionProc.WaitForExit();
sw.Stop();
File.Delete(outputFilename);
outputStr += $"\nconverted in {sw.Elapsed}";
}
return new Tuple<bool, string>(true, outputStr);
}
catch (Exception e)
{
Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
return new Tuple<bool, string>(false, $"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
}
}
public static async Task<bool> SponsorCheck(ttrss.datastructures.Headline hl)
{
if(!hl.link.Host.EndsWith("youtube.com"))
{
//sponsorblock, sadly, only exists for youtube
return true;
}
var match = Regex.Match(hl.link.Query, "v=([^&]+)(&|$)");
var videoId = match.Groups?[1].Value;
var c = new HttpClient();
var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview");
if(sponsorblockcheck.StatusCode == System.Net.HttpStatusCode.NotFound)
{
Console.WriteLine($"sponsorblock reports that {videoId} has no entries (yet)");
var updateTimestamp = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
updateTimestamp = updateTimestamp.AddSeconds(hl.updated).ToLocalTime();
if(DateTime.Now - updateTimestamp > TimeSpan.FromMinutes(45))
{
Console.WriteLine($"updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock");
return true;
}
else
{
Console.WriteLine($"going to wait a bit for segments to show up.");
return false;
}
}
else
{
return true;
}
}
} }
} }

View File

@ -4,16 +4,19 @@
"password": "sordph1sh", "password": "sordph1sh",
"podcastTitlePrefix": "[podcast title] - ", "podcastTitlePrefix": "[podcast title] - ",
"onDoneCopy":"./", "onDoneCopy":"./",
"feedActions": "workingDirectory":"working/",
"chatScript": "miscChat.sh",
"userAgent": "Mozilla/5.0 (compatible; wget-is-not-a-crime/1.0)",
"feedActions":
[ [
{ {
"triggerlabelCaption":"dl plz", "triggerlabelCaption":"dl plz",
"command":"dl" "command":"dl"
}, },
{ {
"triggerlabelCaption":"podcastify-yt plz", "triggerlabelCaption":"podcastify-yt plz",
"command":"podcastifyYT" "command":"podcastifyYT"
}, },
{ {
"triggerlabelCaption":"podcastify-attachment plz", "triggerlabelCaption":"podcastify-attachment plz",
"command":"podcastifyAttachment" "command":"podcastifyAttachment"

16
sponosrblock/Segment.cs Normal file
View File

@ -0,0 +1,16 @@
using System.Collections.Generic;
namespace ttrss_co_client.sponsorblock
{
public class Segment
{
public string category { get; set; }
public string actionType { get; set; }
public double[] segment { get; set; } //start time, end time
public string UUID { get; set; }
public double videoDuration { get; set; } //yes, this is repeated in each segment
public int locked { get; set; }
public int votes { get; set; }
public string description { get; set; }
}
}

25
tasks/ChatMessage.cs Normal file
View File

@ -0,0 +1,25 @@
namespace ttrss_co_client.tasks;
using System.Linq;
using System.Diagnostics;
using ttrss_co_client.ttrss;
using ttrss_co_client.ttrss.datastructures;
using System.Threading.Tasks;
using System;
public class ChatMessage : Phase2Task
{
public static string ChatScript { get; set; }
public override string TaskName => "chatmessage";
public override async Task<Tuple<TaskStatus, WorkOrder>> ActOn(WorkOrder workOrder)
{
await Process.Start(ChatScript,
workOrder.data["chatmessage"] +
(await TtrssClient.GetArticles(workOrder.articleId)).First().feed_title +
": " +
(await TtrssClient.GetArticles(workOrder.articleId)).First().title
).WaitForExitAsync();
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.Done, workOrder);
}
}

51
tasks/Convert.cs Normal file
View File

@ -0,0 +1,51 @@
using System.Diagnostics;
using ttrss_co_client.ttrss;
using ttrss_co_client.ttrss.datastructures;
namespace ttrss_co_client.tasks
{
///<summary>ffmpegify</summary>
public class Convert : Phase2Task
{
public override string TaskName => "convert";
public override async Task<Tuple<TaskStatus, WorkOrder>> ActOn(WorkOrder wo)
{
var conversionFilenameTarget = wo.data["path"].Substring(0, wo.data["path"].LastIndexOf('.')) + wo.data["conversion-target"];
if(conversionFilenameTarget == wo.data["path"])
{
Console.WriteLine("no conversion needed");
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, wo);
}
var article = (await TtrssClient.GetArticles(wo.articleId))?.FirstOrDefault();
if(article == null){
Console.Error.WriteLine($"article {wo.articleId} couldn't be retrieved!");
return null;
}
var sw = new Stopwatch();
sw.Start();
var conversionProc = Process.Start("ffmpeg", $"-y -loglevel quiet -i \"{wo.data["path"]}\" \"{conversionFilenameTarget}\"");
conversionProc.WaitForExit();
sw.Stop();
if(File.Exists(conversionFilenameTarget))
{
Console.WriteLine($" converted {wo.data["path"]} -> {conversionFilenameTarget}");
File.Delete(wo.data["path"]);
wo.data["path"] =conversionFilenameTarget;
}
var outputStr = $"converted in {sw.Elapsed}";
Console.WriteLine(outputStr);
//re-get, in case conversion took a long time
article = (await TtrssClient.GetArticles(wo.articleId))?.FirstOrDefault();
await TtrssClient.UpdateArticleNote($"{article.note}\n{outputStr}", article.id);
var toReturn = new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, wo);
if(!wo.Phase2TaskList.Any())
{
Console.Error.WriteLine($"work order {wo.guid} came to conversion, but thinks its done?");
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.Done, null);
}
return toReturn;
}
}
}

19
tasks/MarkRead.cs Normal file
View File

@ -0,0 +1,19 @@
namespace ttrss_co_client.tasks;
using System.Linq;
using System.Diagnostics;
using ttrss_co_client.ttrss;
using ttrss_co_client.ttrss.datastructures;
using System.Threading.Tasks;
using System;
public class MarkRead : Phase2Task
{
public override string TaskName => "markread";
public override async Task<Tuple<TaskStatus, WorkOrder>> ActOn(WorkOrder workOrder)
{
await TtrssClient.UpdateArticleField(ApiClient.UPDATEFIELD.unread, ApiClient.UPDATEMODE.SetFalse, workOrder.articleId);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
}
}

15
tasks/Phase1Task.cs Normal file
View File

@ -0,0 +1,15 @@
using ttrss_co_client.ttrss;
using ttrss_co_client.ttrss.datastructures;
namespace ttrss_co_client.tasks
{
///<summary>generally, download</summary>
public abstract class Phase1Task
{
public virtual string TaskName { get { return this.GetType().ToString(); } }
public static ApiClient TtrssClient { get; set; }
public static Configuration Conf { get; set; }
public virtual string TriggerLabel { get; set; }
public abstract Task<WorkOrder> ActOn(Headline headline, IEnumerable<Label> labels);
}
}

16
tasks/Phase2Task.cs Normal file
View File

@ -0,0 +1,16 @@
using ttrss_co_client.ttrss;
using ttrss_co_client.ttrss.datastructures;
namespace ttrss_co_client.tasks
{
///<summary>convert, rip ads, etc</summary>
public abstract class Phase2Task
{
public virtual string TaskName { get { return this.GetType().ToString(); } }
public static ApiClient TtrssClient { get; set; }
public static Configuration Conf { get; set; }
public enum TaskStatus {Done, ContinueNow, TryLater}
public abstract Task<Tuple<TaskStatus, WorkOrder>> ActOn(WorkOrder taskSpec);
}
}

View File

@ -0,0 +1,96 @@
using System.Diagnostics;
using ttrss_co_client.ttrss;
using ttrss_co_client.ttrss.datastructures;
namespace ttrss_co_client.tasks
{
///<summary>podcast, with attachment that is podcast</summary>
public class PodcastifyAttachment : Phase1Task
{
public override string TaskName => "podcastifyAttachment";
public override async Task<WorkOrder> ActOn(Headline headline, IEnumerable<Label> labelsWRTArticle)
{
Console.WriteLine($" podcast-is-attachment: {headline.link.ToString()}");
var myGuid = Guid.NewGuid().ToString();
var workingFolder = $"{Conf.WorkingDirectory}/{myGuid}";
var sw = new Stopwatch();
if(!Path.Exists(workingFolder))
{
Directory.CreateDirectory(workingFolder);
}
try
{
var attachmentLink = headline.attachments.Select(a => a.content_url)?.FirstOrDefault().ToString();
var extensionUpstream = attachmentLink.Substring(attachmentLink.LastIndexOf('.'));
var downloadPath = Path.Combine(workingFolder, headline.title) + extensionUpstream;
var downloader = new HttpClient();
downloader.DefaultRequestHeaders.UserAgent.ParseAdd(Conf.UserAgent);
sw.Start();
var dlResult = (await downloader.GetAsync(attachmentLink));
File.WriteAllBytes(downloadPath, await dlResult.Content.ReadAsByteArrayAsync());
sw.Stop();
var outputStr = $"{(dlResult.IsSuccessStatusCode ? "Success" : "fail")} in {sw.Elapsed}";
if(!dlResult.IsSuccessStatusCode)
{
outputStr += $"\n\t{dlResult.StatusCode} - {dlResult.ReasonPhrase}";
}
Console.WriteLine($" {attachmentLink}\n{outputStr}");
await TtrssClient.SetArticleLabel(labelsWRTArticle.First(l => l.caption?.ToLower() == this.TriggerLabel.ToLower()).id, false, headline.id);
Console.WriteLine($" {headline.title}: download trigger label removed");
var podcastTitle = headline.feed_title;
var titlingLabel = labelsWRTArticle.FirstOrDefault(l => l.@checked && l.caption?.ToLower().StartsWith(Conf.PodcastTitlePrefix) == true);
if(titlingLabel != null)
{
await TtrssClient.SetArticleLabel(titlingLabel.id, false, headline.id);
Console.WriteLine($" {headline.title}: podcast titling label removed");
podcastTitle = titlingLabel.caption.Substring(Conf.PodcastTitlePrefix.Length);
}
Console.WriteLine($"article {headline.id} - podcastifying; {podcastTitle}");
await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - podcastify attachment (dl): {outputStr}", headline.id);
if (!dlResult.IsSuccessStatusCode)
{
return null;
}
else
{
Console.WriteLine($"{attachmentLink} downloaded.");
var toReturn = new WorkOrder()
{
articleId = headline.id,
Phase2TaskList = new Dictionary<int, string>(),
data = new Dictionary<string, string>(),
guid = Guid.Parse(myGuid)
};
toReturn.data["path"] = downloadPath;
toReturn.Phase2TaskList[1] = "filemovePublish";
toReturn.data["publish-target"] = $"{Conf.OnDoneCopy}/Podcasts/{podcastTitle}/";
if(extensionUpstream != ".mp3")
{
Console.WriteLine($"{headline.title} needs conversion task.");
toReturn.Phase2TaskList[0] = "convert";
toReturn.data["conversion-target"] = ".mp3";
}
toReturn.Phase2TaskList[2] = "markread";
toReturn.Phase2TaskList[3] = "chatmessage";
toReturn.data["chatmessage"] = "new podcast from ";
return toReturn;
}
}
catch (Exception e)
{
Console.Error.WriteLine($"fatal error in podcast attachment DL for {headline.link}");
Console.Error.WriteLine($"{e.ToString()}: {e.Message}.\n{e.StackTrace}");
await TtrssClient.UpdateArticleNote($"{headline.note}\n[{DateTime.Now.ToString("o")}] - fatal error {e.ToString()}: {e.Message}.\n{e.StackTrace}", headline.id);
return null;
}
}
}
}

59
tasks/PodcastifyYT.cs Normal file
View File

@ -0,0 +1,59 @@
using System.Diagnostics;
using ttrss_co_client.ttrss;
using ttrss_co_client.ttrss.datastructures;
namespace ttrss_co_client.tasks
{
///<summary>download from YT</summary>
public class PodcastifyYT : Phase1Task
{
public override string TaskName => "podcastifyYT";
public override async Task<WorkOrder> ActOn(Headline headline, IEnumerable<Label> labelsWRTArticle)
{
Console.WriteLine($" YT podcastify: {headline.link.ToString()}");
var myGuid = Guid.NewGuid().ToString();
var toReturn = new WorkOrder()
{
articleId = headline.id,
Phase2TaskList = new Dictionary<int, string>(),
data = new Dictionary<string, string>(),
guid = Guid.Parse(myGuid)
};
var podcastTitle = headline.feed_title;
var titlingLabel = labelsWRTArticle.FirstOrDefault(l => l.@checked && l.caption?.ToLower().StartsWith(Conf.PodcastTitlePrefix) == true);
if(titlingLabel != null)
{
podcastTitle = titlingLabel.caption.Substring(Conf.PodcastTitlePrefix.Length);
await TtrssClient.SetArticleLabel(titlingLabel.id, false, headline.id);
}
Console.WriteLine($"article {headline.id} - yt podcastifying; {podcastTitle}");
toReturn.Phase2TaskList[0] = "yt-dlp";
toReturn.data["ytdlp-link"] = headline.link.ToString();
toReturn.Phase2TaskList[1] = "convert";
toReturn.data["conversion-target"] = ".mp3";
if(headline.link.Host.EndsWith("youtube.com"))
{
toReturn.Phase2TaskList[2] = "sponsorblock";
}
toReturn.Phase2TaskList[3] = "filemovePublish";
toReturn.data["publish-target"] = $"{Conf.OnDoneCopy}/Podcasts/{podcastTitle}/";
toReturn.Phase2TaskList[4] = "markread";
toReturn.Phase2TaskList[5] = "chatmessage";
toReturn.data["chatmessage"] = "new podcast (via YT) from ";
await TtrssClient.SetArticleLabel(labelsWRTArticle.First(l => l.caption?.ToLower() == this.TriggerLabel.ToLower()).id, false, headline.id);
Console.WriteLine($" {headline.title}: download trigger label removed");
return toReturn;
}
}
}

32
tasks/Publish.cs Normal file
View File

@ -0,0 +1,32 @@
using System.Linq;
using System.Diagnostics;
using ttrss_co_client.ttrss;
using ttrss_co_client.ttrss.datastructures;
namespace ttrss_co_client.tasks
{
///<summary>Move to output</summary>
public class Publish : Phase2Task
{
public override string TaskName => "filemovePublish";
public override async Task<Tuple<TaskStatus, WorkOrder>>ActOn(WorkOrder workOrder)
{
var targetDirectory = workOrder.data["publish-target"];
if(!Directory.Exists(targetDirectory))
{
Directory.CreateDirectory(targetDirectory);
}
var outputFilename = Path.Combine(workOrder.data["publish-target"], Path.GetFileName(workOrder.data["path"]));
foreach(char c in "'\":\\?")
{
outputFilename = outputFilename.Replace(c, ' ');
}
File.Move(workOrder.data["path"], outputFilename, true);
var article = (await TtrssClient.GetArticles(workOrder.articleId))?.FirstOrDefault();
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] - copied", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
}
}
}

127
tasks/Sponsorblock.cs Normal file
View File

@ -0,0 +1,127 @@
using System;
using System.Text;
using Newtonsoft.Json;
using System.Linq;
using System.Diagnostics;
using System.Text.RegularExpressions;
using ttrss_co_client.ttrss;
using ttrss_co_client.ttrss.datastructures;
namespace ttrss_co_client.tasks
{
public class Sponsorblock : Phase2Task
{
public override string TaskName => "sponsorblock";
public override async Task<Tuple<TaskStatus, WorkOrder>>ActOn(WorkOrder workOrder)
{
var sw = new Stopwatch();
sw.Start();
var article = (await TtrssClient.GetArticles(workOrder.articleId)).FirstOrDefault();
if(article == null)
{
Console.Error.WriteLine($"couldn't find article {workOrder.articleId} for workOrder {workOrder.guid}?!");
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.Done, null);
}
if (!article.link.Host.EndsWith("youtube.com"))
{
Console.WriteLine("no sponsorblock segments on sites other than YT");
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
}
var match = Regex.Match(article.link.Query, "v=([^&]+)(&|$)");
var videoId = match.Groups?[1].Value;
var c = new HttpClient();
var sponsorblockcheck = await c.GetAsync($"https://sponsor.ajay.app/api/skipSegments?videoID={videoId}&category=sponsor&category=selfpromo&category=interaciton&category=intro&category=outro&category=preview&category=interaction");
IEnumerable<sponsorblock.Segment> segments = null;
try
{
segments = JsonConvert.DeserializeObject<IEnumerable<sponsorblock.Segment>>(await sponsorblockcheck.Content.ReadAsStringAsync());
}
catch (Exception e)
{
segments = null;
}
if (sponsorblockcheck.StatusCode == System.Net.HttpStatusCode.NotFound || segments == null || segments.Count() == 0)
{
Console.WriteLine($"sponsorblock reports that {videoId} has no entries (yet)");
var updateTimestamp = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
updateTimestamp = updateTimestamp.AddSeconds(article.updated).ToLocalTime();
if (DateTime.Now - updateTimestamp > TimeSpan.FromMinutes(45))
{
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] updated {updateTimestamp} (more than 45 minutes ago), going to give up waiting for sponsorblock", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
}
else
{
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] waiting for sponsorblock segments", article.id);
workOrder.Phase2TaskList[workOrder.Phase2TaskList.Keys.Min() - 1] = this.TaskName;
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.TryLater, workOrder);
}
}
else
{
Console.WriteLine($"[{DateTime.Now.ToString("o")}] sponsorblock reports {segments.Count()} junk segments");
var contentSegments = new List<Tuple<double, double>>();
var previousEdge = 0.0;
var extension = workOrder.data["path"].Substring(workOrder.data["path"].LastIndexOf('.'));
var intermediatePathBase = workOrder.data["path"].Substring(0, workOrder.data["path"].LastIndexOf('.'));
var conversionProcesses = new List<Tuple<Process, Tuple<int, string>>>();
foreach(var junkSegment in segments.OrderBy(s => s.segment[0]))
{
contentSegments.Add(new Tuple<double, double>(previousEdge, junkSegment.segment[0]));
previousEdge = junkSegment.segment[1];
}
contentSegments.Add(new Tuple<double, double>(previousEdge, segments.First().videoDuration));
contentSegments = contentSegments.Except(contentSegments.Where(tup => tup.Item2 - tup.Item1 < 0.5)).ToList();
#region ffmpeg via intermediate files
var intermediateCount = 0;
foreach(var seg in contentSegments)
{
var intermediateTargetPath = $"{intermediatePathBase}-intermediate-{intermediateCount.ToString("D2")}{extension}";
conversionProcesses.Add(new Tuple<Process, Tuple<int, string>>(
Process.Start("ffmpeg", $"-y -loglevel quiet -i \"{workOrder.data["path"]}\" -ss {seg.Item1} -to {seg.Item2} \"{intermediateTargetPath}\""),
new Tuple<int, string>(intermediateCount, intermediateTargetPath)
));
Console.WriteLine("waiting for exit from task, for debugging reasons");
conversionProcesses.Last().Item1.WaitForExit();
intermediateCount++;
}
Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments being exported");
var intermediates = new Dictionary<int, string>();
foreach(var proc in conversionProcesses)
{
proc.Item1.WaitForExit();
intermediates[proc.Item2.Item1] = proc.Item2.Item2;
}
Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments should be exported, stitching together");
var sb = new StringBuilder();
sb.AppendLine("#ffmpeg demands it");
foreach(var intermediate in intermediates.OrderBy(kvp => kvp.Key))
{
sb.AppendLine($"file '{intermediate.Value}'");
}
var ffmpegFile = Path.Combine(Path.GetDirectoryName(workOrder.data["path"]), "ffmpeglist.txt");
File.WriteAllText(ffmpegFile, sb.ToString());
Process.Start("ffmpeg", $"-y -f concat -safe 0 -i {ffmpegFile} -c copy \"{workOrder.data["path"]}\"")
.WaitForExit();
File.Delete(ffmpegFile);
Console.WriteLine($"[{DateTime.Now.ToString("o")}] intermediate content segments stitched. Deleting originals.");
foreach(var intermediate in intermediates.Values)
{
File.Delete(intermediate);
}
#endregion
sw.Stop();
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] removed {segments.Count()} junk segments", article.id);
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
}
}
}
}

49
tasks/StandardDL.cs Normal file
View File

@ -0,0 +1,49 @@
using System.Diagnostics;
using ttrss_co_client.ttrss;
using ttrss_co_client.ttrss.datastructures;
namespace ttrss_co_client.tasks
{
///<summary>download from YT</summary>
public class StandardDL : Phase1Task
{
public override string TaskName => "dl";
public override async Task<WorkOrder> ActOn(Headline headline, IEnumerable<Label> labelsWRTArticle)
{
Console.WriteLine($" standard download: {headline.link.ToString()}");
var myGuid = Guid.NewGuid().ToString();
var toReturn = new WorkOrder()
{
articleId = headline.id,
Phase2TaskList = new Dictionary<int, string>(),
data = new Dictionary<string, string>(),
guid = Guid.Parse(myGuid)
};
toReturn.Phase2TaskList[0] = "yt-dlp";
toReturn.data["ytdlp-link"] = headline.link.ToString();
toReturn.Phase2TaskList[1] = "convert";
toReturn.data["conversion-target"] = ".mp4";
if(headline.link.Host.EndsWith("youtube.com"))
{
toReturn.Phase2TaskList[2] = "sponsorblock";
}
toReturn.Phase2TaskList[3] = "filemovePublish";
toReturn.data["publish-target"] = $"{Conf.OnDoneCopy}/recent episodes/";
toReturn.Phase2TaskList[4] = "markread";
toReturn.Phase2TaskList[5] = "chatmessage";
toReturn.data["chatmessage"] = "new video from ";
await TtrssClient.SetArticleLabel(labelsWRTArticle.First(l => l.caption?.ToLower() == this.TriggerLabel.ToLower()).id, false, headline.id);
Console.WriteLine($" {headline.title}: download trigger label removed");
return toReturn;
}
}
}

12
tasks/WorkOrder.cs Normal file
View File

@ -0,0 +1,12 @@
using System.Collections.Generic;
namespace ttrss_co_client.tasks
{
public class WorkOrder
{
public int articleId { get; set; }
public Guid guid { get; set; }
public Dictionary<int, string> Phase2TaskList { get; set; } //so youtube downloads will download, then add a conversion to this, then add an adblock, then publish
public Dictionary<string, string> data { get; set; }
}
}

56
tasks/Ytdlp.cs Normal file
View File

@ -0,0 +1,56 @@
using System;
using System.Text;
using Newtonsoft.Json;
using System.Linq;
using System.Diagnostics;
using System.Text.RegularExpressions;
using ttrss_co_client.ttrss;
using ttrss_co_client.ttrss.datastructures;
namespace ttrss_co_client.tasks
{
///<summary>Move to output</summary>
public class Ytdlp : Phase2Task
{
public override string TaskName => "yt-dlp";
public override async Task<Tuple<TaskStatus, WorkOrder>>ActOn(WorkOrder workOrder)
{
var article = (await TtrssClient.GetArticles(workOrder.articleId)).FirstOrDefault();
var sw = new Stopwatch();
var ytdl = new YoutubeDLSharp.YoutubeDL();
ytdl.YoutubeDLPath = "yt-dlp";
ytdl.FFmpegPath = "ffmpeg";
ytdl.OutputFolder = $"{Conf.WorkingDirectory}/{workOrder.guid}";
ytdl.OutputFileTemplate = "%(upload_date)s - %(title)s - [%(id)s].%(ext)s";
if(!Path.Exists(ytdl.OutputFolder))
{
Directory.CreateDirectory(ytdl.OutputFolder);
}
sw.Start();
var res = await ytdl.RunVideoDownload(workOrder.data["ytdlp-link"]);
sw.Stop();
if (!res.Success)
{
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] yt-dlp fatal error. \n{string.Join('\n', res.ErrorOutput)}", article.id);
return null;
}
await TtrssClient.UpdateArticleNote($"{article.note}\n[{DateTime.Now.ToString("o")}] yt-dlp {(res.Success ? "success" : "fail")} in {sw.Elapsed}", article.id);
var outputFilename = res.Data;
foreach(char c in "'\"")
{
outputFilename = outputFilename.Replace(c, '_');
}
if(outputFilename != res.Data)
{
File.Move(res.Data, outputFilename);
}
workOrder.data["path"] = outputFilename;
return new Tuple<TaskStatus, WorkOrder>(TaskStatus.ContinueNow, workOrder);
}
}
}