seems to theoretically work

This commit is contained in:
Adam R. Grey 2021-06-19 01:07:12 -04:00
parent 92c5112f4d
commit c3fdf8fb9b
24 changed files with 949 additions and 0 deletions

63
franz.tests/UnitTest1.cs Normal file
View File

@ -0,0 +1,63 @@
using System;
using System.Threading.Tasks;
using franz;
using NUnit.Framework;
namespace franz.tests
{
[TestFixture]
public class Tests
{
Telefranz f1;
Telefranz f2;
[SetUp]
public void Setup()
{
f1 = new Telefranz("libfranztest1", "focalor:9092", "/usr/lib/librdkafka.so");
f2 = new Telefranz("libfranztest2", "focalor:9092", "/usr/lib/librdkafka.so");
}
[Test]
public void reportsToSoundoff()
{
var f1Reported = false;
var f2Reported = false;
f1.addHandler((silver_messages.global.report r) => {
switch(r.name)
{
case "libfranztest1":
{
f1Reported = true;
TestContext.Out.WriteLine("f1 reported");
break;
}
case "libfranztest2":
{
f2Reported = true;
TestContext.Out.WriteLine("f2 reported");
break;
}
}
});
f1.StartListening();
f2.StartListening();
f1.ProduceMessage(new silver_messages.global.sound_off());
Task.WaitAny(
Task.Run(async () => {
while(f1Reported == false || f2Reported == false)
{
//TestContext.Out.WriteLine("not ready, giving another 100ms");
await Task.Delay(100);
}
//TestContext.Out.WriteLine("done, ready, green");
}),
Task.Run(async () => {
await Task.Delay(10000);
TestContext.Out.WriteLine("time up");
Assert.Fail();
})
);
}
}
}

View File

@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net5.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="NUnit" Version="3.12.0" />
<PackageReference Include="NUnit3TestAdapter" Version="3.16.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\franz\franz.csproj" />
</ItemGroup>
</Project>

375
franz/.gitignore vendored Normal file
View File

@ -0,0 +1,375 @@
# ---> VisualStudio
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
##
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
# User-specific files
*.rsuser
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Mono auto generated files
mono_crash.*
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
[Ww][Ii][Nn]32/
[Aa][Rr][Mm]/
[Aa][Rr][Mm]64/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
[Ll]ogs/
# Visual Studio 2015/2017 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# Visual Studio 2017 auto generated files
Generated\ Files/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUnit
*.VisualState.xml
TestResult.xml
nunit-*.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# Benchmark Results
BenchmarkDotNet.Artifacts/
# .NET Core
project.lock.json
project.fragment.lock.json
artifacts/
# ASP.NET Scaffolding
ScaffoldingReadMe.txt
# StyleCop
StyleCopReport.xml
# Files built by Visual Studio
*_i.c
*_p.c
*_h.h
*.ilk
*.meta
*.obj
*.iobj
*.pch
*.pdb
*.ipdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*_wpftmp.csproj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
*.VC.db
*.VC.VC.opendb
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# Visual Studio Trace Files
*.e2e
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# AxoCover is a Code Coverage Tool
.axoCover/*
!.axoCover/settings.json
# Coverlet is a free, cross platform Code Coverage Tool
coverage*.json
coverage*.xml
coverage*.info
# Visual Studio code coverage results
*.coverage
*.coveragexml
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# Note: Comment the next line if you want to checkin your web deploy settings,
# but database connection strings (with potential passwords) will be unencrypted
*.pubxml
*.publishproj
# Microsoft Azure Web App publish settings. Comment the next line if you want to
# checkin your Azure Web App publish settings, but sensitive information contained
# in these scripts will be unencrypted
PublishScripts/
# NuGet Packages
*.nupkg
# NuGet Symbol Packages
*.snupkg
# The packages folder can be ignored because of Package Restore
**/[Pp]ackages/*
# except build/, which is used as an MSBuild target.
!**/[Pp]ackages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/[Pp]ackages/repositories.config
# NuGet v3's project.json files produces more ignorable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directories and files
AppPackages/
BundleArtifacts/
Package.StoreAssociation.xml
_pkginfo.txt
*.appx
*.appxbundle
*.appxupload
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!?*.[Cc]ache/
# Others
ClientBin/
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.jfm
*.pfx
*.publishsettings
orleans.codegen.cs
# Including strong name files can present a security risk
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
#*.snk
# Since there are multiple workflows, uncomment next line to ignore bower_components
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
#bower_components/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
ServiceFabricBackup/
*.rptproj.bak
# SQL Server files
*.mdf
*.ldf
*.ndf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
*.rptproj.rsuser
*- [Bb]ackup.rdl
*- [Bb]ackup ([0-9]).rdl
*- [Bb]ackup ([0-9][0-9]).rdl
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
node_modules/
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
*.vbw
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
paket-files/
# FAKE - F# Make
.fake/
# CodeRush personal settings
.cr/personal
# Python Tools for Visual Studio (PTVS)
__pycache__/
*.pyc
# Cake - Uncomment if you are using it
# tools/**
# !tools/packages.config
# Tabs Studio
*.tss
# Telerik's JustMock configuration file
*.jmconfig
# BizTalk build output
*.btp.cs
*.btm.cs
*.odx.cs
*.xsd.cs
# OpenCover UI analysis results
OpenCover/
# Azure Stream Analytics local run output
ASALocalRun/
# MSBuild Binary and Structured Log
*.binlog
# NVidia Nsight GPU debugger configuration file
*.nvuser
# MFractors (Xamarin productivity tool) working folder
.mfractor/
# Local History for Visual Studio
.localhistory/
# BeatPulse healthcheck temp database
healthchecksdb
# Backup folder for Package Reference Convert tool in Visual Studio 2017
MigrationBackup/
# Ionide (cross platform F# VS Code tools) working folder
.ionide/
# Fody - auto-generated XML schema
FodyWeavers.xsd
# ---> VisualStudioCode
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
*.code-workspace
# Local History for Visual Studio Code
.history/

10
franz/README.md Normal file
View File

@ -0,0 +1,10 @@
# franz
allows you to speak kafka
## spec
* snake_case for all "title"-y things.
* every message type should have a type, which is a string (in snake case), which cannot be null. Everything else varies by topic.
Everyone on kafka must subscribe to the silver_meddlists.global topic. - ideal if franz itself can handle all the minimum requirements.

187
franz/Telefranz.cs Normal file
View File

@ -0,0 +1,187 @@
using Confluent.Kafka;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
namespace franz
{
public class Telefranz : IDisposable
{
private string handling_group { get; set; } = "Liszt";
const string namespace_prefix = "silver_messages";
private silver_messages.global.report howToReport { get; set; }
private bool shouldListen = false;
private IProducer<Null, string> producer { get; set; }
private IConsumer<Ignore, string> consumer { get; set; }
private static List<Type> allMessageTypes { get; set; } = null;
private Task listenTask = null;
private Dictionary<Type, List<Action<silver_messages.message>>> handlers { get; set; }
private Dictionary<object, Action<silver_messages.message>> handlerWrappings
= new Dictionary<object, Action<silver_messages.message>>();
public Telefranz(string name, string bootstrap_servers, string rdkafka_location,
List<string> commands = null, List<string> checks = null,
List<string> errors = null, List<string> warnings = null)
{
Library.Load(rdkafka_location);
this.handling_group = name;
this.howToReport = new silver_messages.global.report()
{
name = name,
errors = errors ?? new List<string>(),
warnings = warnings ?? new List<string>()
};
if (commands?.Count > 0)
{
howToReport.capabilites.commands = commands;
}
if (checks?.Count > 0)
{
howToReport.capabilites.checks = checks;
}
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrap_servers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrap_servers,
GroupId = this.handling_group
};
producer = new ProducerBuilder<Null, string>(producerConfig).Build();
consumer = new ConsumerBuilder<Ignore, string>(consumerConfig).Build();
staticSetup();
handlers = new Dictionary<Type, List<Action<silver_messages.message>>>();
foreach (var messageType in allMessageTypes)
{
handlers.Add(messageType, new List<Action<silver_messages.message>>());
}
addHandler<silver_messages.global.sound_off>((m) =>
{
ProduceMessage(this.howToReport);
});
addHandler<silver_messages.global.stop>((m) =>
{
if (m.name == this.handling_group)
{
Environment.Exit(0);
}
});
addHandler<silver_messages.global.restart>((m) =>
{
if (m.name == this.handling_group)
{
throw new NotImplementedException();//TODO
}
});
}
public void addHandler<t>(Action<t> theAction) where t : silver_messages.message
{
Action<silver_messages.message> wrapped = (m) =>
{
theAction(m as t);
};
handlerWrappings.Add(theAction, wrapped);
if (handlers[typeof(t)].Count == 0)
{
var sublist = consumer.Subscription;
sublist.Add(typeof(t).ToString());
consumer.Subscribe(sublist);
}
handlers[typeof(t)].Add(wrapped);
}
public void removeHandler<t>(Action<t> theAction) where t : silver_messages.message
{
handlers[typeof(t)].Remove(handlerWrappings[theAction as System.Action]);
handlerWrappings.Remove(theAction as System.Action);
if (handlers[typeof(t)].Count == 0)
{
var sublist = consumer.Subscription;
consumer.Unsubscribe();
sublist.Remove(typeof(t).ToString());
consumer.Subscribe(sublist);
}
}
public void ProduceMessage(silver_messages.message message)
{
var msgText = JsonConvert.SerializeObject(message, type: message.GetType(), null);
producer.Produce(message.GetType().ToString(), new Message<Null, string> { Value = msgText });
}
public void StartListening()
{
StopListening();
listenTask = Task.Run(() =>
{
try
{
shouldListen = true;
while (shouldListen)
{
var cr = consumer.Consume(125);
if (cr != null)
{
var typeHaver = JsonConvert.DeserializeAnonymousType(cr.Message.Value, new { type = "" });
var messageType = handlers.Keys.FirstOrDefault(hkey => hkey.ToString() == typeHaver.type && handlers[hkey].Count > 0);
if (messageType != null)
{
var message = JsonConvert.DeserializeObject(cr.Message.Value, messageType);
foreach (var handling in handlers[messageType])
{
handling(message as silver_messages.message);
}
}
}
}
}
catch (System.OperationCanceledException)
{
Console.WriteLine("cancelled");
}
catch (Confluent.Kafka.ConsumeException e)
{
Console.Error.WriteLine(JsonConvert.SerializeObject(e));
throw;
}
});
}
public void StopListening()
{
shouldListen = false;
}
private static void staticSetup()
{
if (allMessageTypes != null)
{
return;
}
allMessageTypes = new List<Type>();
foreach (var messageType in from t in Assembly.GetExecutingAssembly().GetTypes()
where t.IsClass && t.Namespace?.StartsWith(namespace_prefix) == true
select t)
{
allMessageTypes.Add(messageType);
}
}
public void Dispose()
{
StopListening();
producer.Dispose();
consumer.Dispose();
}
}
}

16
franz/franz.csproj Normal file
View File

@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net5.0</TargetFramework>
<PackageId>franz</PackageId>
<Version>0.0.1</Version>
<Authors>adam</Authors>
<Company>Silver Meddlists</Company>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Confluent.Kafka" Version="1.7.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
</ItemGroup>
</Project>

View File

@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using silver_messages;
namespace silver_messages.directorial
{
public class check_complete : message
{
public string check { get; set; }
public string result { get; set; }
}
}

View File

@ -0,0 +1,18 @@
using System;
using System.Collections.Generic;
using silver_messages;
namespace silver_messages.directorial
{
public class command_completed : message
{
//name of the command that was called
public string command { get; set; }
//runtime in ms
public uint runtime { get; set; }
public string stdout { get; set; }
public string stderr { get; set; }
public int exit_code { get; set; }
}
}

View File

@ -0,0 +1,15 @@
using System;
using System.Collections.Generic;
using silver_messages;
namespace silver_messages.directorial
{
public class command_error : message
{
//name of the command that was called
public string command { get; set; }
//runtime so far in ms
public uint runtime { get; set; }
public string stderr { get; set; }
}
}

View File

@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using silver_messages;
namespace silver_messages.directorial
{
public class command_expired : message
{
//name of the command that was called
public string command { get; set; }
//runtime in ms
public uint runtime { get; set; }
}
}

View File

@ -0,0 +1,15 @@
using System;
using System.Collections.Generic;
using silver_messages;
namespace silver_messages.directorial
{
public class command_output : message
{
//name of the command that was called
public string command { get; set; }
//runtime so far in ms
public uint runtime { get; set; }
public string stdout { get; set; }
}
}

View File

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using silver_messages;
namespace silver_messages.directorial
{
/*
* run it and send a check_complete
*/
public class execute_check : message
{
public string check { get; set; }
public List<string> args { get; set; } = new List<string>();
}
}

View File

@ -0,0 +1,22 @@
using System;
using System.Collections.Generic;
using silver_messages;
namespace silver_messages.directorial
{
/*
* if this is the name of one of your capabilities, pass args to it and execute it.
* if timeout is > 0, wait that long (in seconds) before terminating the command.
* if the command completed, send a command_completed. if you had to kill it because
* timeout, send command_expired. If it has no timeout, send command_output and
* command_error as you get them, and if it ends some other way, send command_completed
* with empty stdout and stderr.
*/
public class execute_command : message
{
public string command { get; set; }
public List<string> args { get; set; } = new List<string>();
public int timeout { get; set; }
}
}

View File

@ -0,0 +1,36 @@
using System;
using System.Collections.Generic;
namespace silver_messages.global
{
public class report : message
{
//your name
public string name { get; set; }
//capabilities by name which aren't implied by your name
public Capabilites capabilites { get; set; } = new Capabilites();
//reasons you can't fill capabilities, implied or stated. Ideally with a stack trace.
public List<string> errors { get; set; } = new List<string>();
//warnings about your capabilities
public List<string> warnings { get; set; } = new List<string>();
public status GetStatus()
{
if (errors?.Count > 0)
return status.Errors;
else if (warnings?.Count > 0)
return status.Warnings;
else
return status.Good;
}
public class Capabilites
{
public IEnumerable<string> commands { get; set; }
public IEnumerable<string> checks { get; set; }
}
public enum status { Good, Warnings, Errors };
}
}

View File

@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;
using silver_messages;
namespace silver_messages.global
{
public class restart : message
{
//check if this is you (i.e., your handling group). If so, restart. Yourself, not the hardware.
public string name { get; set; }
}
}

View File

@ -0,0 +1,10 @@
using System;
using System.Collections.Generic;
using silver_messages;
namespace silver_messages.global
{
//if you receive this, respond with a report
public class sound_off : message { }
}

View File

@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;
using silver_messages;
namespace silver_messages.global
{
public class stop : message
{
//check if this is you (i.e., your handling group). If so, stop. Yourself, not the hardware.
public string name { get; set; }
}
}

View File

@ -0,0 +1,15 @@
using System;
using System.Collections.Generic;
using Newtonsoft.Json;
namespace silver_messages
{
public abstract class message
{
public string type { get { return this.GetType().ToString(); } }
public override string ToString()
{
return JsonConvert.SerializeObject(this);
}
}
}

View File

@ -0,0 +1,15 @@
using System;
using System.Collections.Generic;
using silver_messages;
namespace silver_messages.youtube
{
/*
* for all the scheduled, not-yet released videos, what required metadata
* do they need
*/
public class metadata_needed : message
{
//key is a yt id
public Dictionary<string, yt_metadata> needed{get;set;} = new Dictionary<string, yt_metadata>();
}
}

View File

@ -0,0 +1,11 @@
using System;
using System.Collections.Generic;
using silver_messages;
namespace silver_messages.youtube
{
/*
* check all the scheduled, not-yet released videos for required metadata.
* respond with a metadata_needed - empty if none.
*/
public class request_metadata_needed : message { }
}

View File

@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using silver_messages;
namespace silver_messages.youtube
{
/*
* someone will send as much metadata as they have - set non-null fields to the provided values
*/
public class update_metadata : message
{
public string yt_id { get; set; }
public yt_metadata metadata { get; set; }
}
}

View File

@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using silver_messages;
namespace silver_messages.youtube
{
/** if you receive this, start the upload (of filename with metadata)
* and then respond with an upload_started
*/
public class upload : message
{
public string filename { get; set; }
public yt_metadata metadata { get; set; }
}
}

View File

@ -0,0 +1,11 @@
using System;
using System.Collections.Generic;
using silver_messages;
namespace silver_messages.youtube
{
public class upload_started : message
{
public string filename { get; set; }
public string yt_id { get; set; }
}
}

View File

@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using silver_messages;
namespace silver_messages.youtube
{
public class yt_metadata
{
//not including (part _)
public string title { get; set; }
public DateTime release { get; set; }
public DateTime recording { get; set; }
//name of the YT playlist to add this to
public string playlist { get; set; }
//my path to the thumbnail - meaning it will need the number stamped some other way
public string thumbnail { get; set; }
}
}