omg it works.

I think I figured it out. your -s << 'ENDMARKER' is looking for a line that starts with ENDMARKER - no spaces. so you can't indent it nicely.

scripts/1clickservice.groovy

@@ -3,16 +3,20 @@ pipeline {
     parameters { 
         string(name: 'servicename', description: "service name")
         string(name: 'svcdesc', description: "service description")
-        boolean(name: 'database', description: "service has a database", defaultValue: true)
+        string(name: 'targetHost', description: "system to live on", defaultValue: "moloryb.lan")
+        booleanParam(name: 'database', description: "service has a database", defaultValue: true)
     }
     environment {
-        pw_linuxserviceaccount=""
-        pw_productiondatabase=""
-        pw_developmentdatabase=""
-        ALLOCES = credentials('//TODO: its usually a uuid')
+        SUDOER_ALLOCES = credentials('a674f816-2b35-4d60-ba60-7b66e86f3c5c')
+        SUDOER_MOLORYB = credentials('1f3b965e-bcc0-4074-99f2-b64dddbf7de7')
+        SUDOERSSHID = '2c48e1a9-22b2-455c-9959-6b29e86d3fb5'
+        SUDOERSSH = credentials('2c48e1a9-22b2-455c-9959-6b29e86d3fb5')
+        JENKINS = credentials('68391381-e095-4b47-b956-d23055b0808e')
+        GITEATOKEN = credentials('d0e86441-2157-405f-8539-a9a9010c6ecf')
+        GITEA_USR='jenkins'
     }
     stages {
-        stage("type strengthening") {
+        stage("environment setup") {
             steps {
                 script {
                     if (servicename.isEmpty()) {
@@ -21,35 +25,212 @@ pipeline {
                     if (servicename.contains(' ')) {
                         error("servicename cannot have spaces. try dashes.")
                     }
-                    sh env.pw_linuxserviceaccount=$(mktemp -u XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX)
-                    sh env.pw_productiondatabase=$(mktemp -u XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX)
-                    sh env.pw_developmentdatabase=$(mktemp -u XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX)
-                    //TODO: save them somewhere. probably better to not lock myself out of these accounts from moment 0 
+
+                    switch (targetHost) {
+                        case "alloces.lan":
+                            SUDOER_USR = SUDOER_ALLOCES_USR
+                            SUDOER_PSW = SUDOER_ALLOCES_PSW
+                        case "moloryb.lan":
+                            SUDOER_USR=SUDOER_MOLORYB_USR
+                            SUDOER_PSW=SUDOER_MOLORYB_PSW
+                      break
+                        default:
+                            error("target host not recognized. btw: yes .lan, all lowercase.")
+                    }
+
+                    env.pw_linuxserviceaccount=sh(returnStdout: true, script:  "mktemp -u XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX").trim()
+                    echo env.pw_linuxserviceaccount
+                    env.pw_productiondatabase=sh(returnStdout: true, script: "mktemp -u XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX").trim()
+                    echo env.pw_productiondatabase
+                    env.pw_developmentdatabase=sh(returnStdout: true, script: "mktemp -u XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX").trim()
+                    echo env.pw_developmentdatabase
                 }
             }
         }
         stage("gitea project"){
             steps{
-                //TODO: clone _template-service. Must be under greyn.
-                //TODO: if not database version, strip out database stuff
+                sh """
+                    curl -X 'POST' \
+                      'https://gitea.arg.rip/api/v1/repos/greyn/_template-service/generate' \
+                      -H 'accept: application/json' \
+                      -H 'Authorization: token ${env.GITEATOKEN}' \
+                      -H 'Content-Type: application/json' \
+                      -d '{
+                          "description": "${svcdesc}",
+                          "git_content": true,
+                          "git_hooks": true,
+                          "labels": true,
+                          "name": "${servicename}",
+                          "owner": "greyn",
+                          "private": false,
+                          "protected_branch": true,
+                          "topics": true,
+                          "webhooks": true
+                        }'
+                """
             }
         }
         stage("jenkins pipeline"){
             steps{
-                //TODO: tell jenkins to scan greyn pipeline
-                //TODO: find this new service in jenkins
-                //TODO: add the shared secrets to jenkins
+                //the bad news is that it looks like it's not allowed to trigger just any old job remotely
+                //the good news is that this seems to pick it up pretty reliably
+                //sh """
+                //    curl -X POST -L --user ${env.JENKINS_USR}:${env.JENKINS_PSW} \
+                //        alloces.lan:8080/job/gitea.arg.rip/build
+                //    """
+
+                timeout(time: 5, unit: 'MINUTES') {
+                    sh """
+                        strRes=""
+                        while [ -z "\$strRes" ]; 
+                        do
+                            sleep 5;
+                            #curl -X GET    ${env.JENKINS_USR}:${env.JENKINS_PSW}@alloces.lan:8080/job/gitea.arg.rip/api/json
+                            curl -X GET -s ${env.JENKINS_USR}:${env.JENKINS_PSW}@alloces.lan:8080/job/gitea.arg.rip/api/json > f.json
+                            
+                            strRes=\$(jq '.jobs' f.json | jq '.[] | select(.name==\"${env.servicename}\")')
+                            rm f.json
+                        done
+                    """ 
+                }
+                withCredentials([sshUserPrivateKey(credentialsId: 'f42347e9-e3b5-44af-a1af-c5e7b9775fee', keyFileVariable: 'PK')]) {
+                    sh """#!/bin/bash
+
+                        #mkdir -p ~/.ssh
+                        ssh-keyscan -t ed25519 gitea.arg.rip >> ~/.ssh/known_hosts
+                        #cat ~/.ssh/known_hosts
+                        #shit doesn't work. ssh in, git clone, get your shit set up for keys.
+                        git -c core.sshCommand="ssh -i '$PK'\" clone ssh://git@gitea.arg.rip:8022/greyn/${servicename}.git
+                        #fyi, future me: pushd is useless here, it pops out. Like it's loading a script and exiting.
+                    """
+                    script { //there's no "if" "step" so any "if" must be in a "script" step
+                        if(database){
+                            sh """#!/bin/bash
+
+                                pushd ${servicename}
+                                dbstartline=\$(sed -n '/---dbstart---/=' Jenkinsfile)
+                                dbendline=\$(sed -n '/---dbend---/=' Jenkinsfile)
+                                echo \"yes db.\"
+                                
+                                sed -i \"\${dbstartline}d;\${dbendline}d\" Jenkinsfile
+
+                                databasecredsid=\$(uuidgen)
+                                httpBasicAuth=\"http://${env.JENKINS_USR}:${env.JENKINS_PSW}@alloces.lan:8080/\"
+                                echo \"\${httpBasicAuth}\"
+                                urlGetData=\"crumbIssuer/api/xml?xpath=concat(//crumbRequestField,\\":\\",//crumb)\"
+                                
+                                CRUMB=\$(curl -s -c cookies.txt \"\${httpBasicAuth}\${urlGetData}\")
+                                echo "crumb anyway. \$CRUMB"
+                                curl -H \$CRUMB -X POST \"http://${env.JENKINS_USR}:${env.JENKINS_PSW}@alloces.lan:8080/job/gitea.arg.rip/job/${servicename}/credentials/store/folder/domain/_/createCredentials\" \
+                                --data-urlencode 'json={
+                                  "": "0",
+                                  "credentials": {
+                                    "scope": "GLOBAL",
+                                    "id": "'"\$databasecredsid"'",
+                                    "secret": "Host=${targetHost};Database=${servicename};Username=${servicename};Password=${env.pw_productiondatabase};IncludeErrorDetail=true;",
+                                    "description": "database connection string",
+                                    "\$class": "com.cloudbees.plugins.credentials.impl.UsernamePasswordCredentialsImpl"
+                                  }
+                                }'
+                                sed -i 's/productiondatabase_connectionString=creds/productiondatabase_connectionString=credentials('\$databasecredsid')/\' Jenkinsfile
+
+                                rm cookies.txt
+                                git add .
+                                git commit -m \"set up for database\"
+                            """
+                        }
+                        else{
+                            sh """#!/bin/bash
+                                echo \"no db\"
+                                pushd ${servicename}
+                                sed -i '\${dbstartline},\${dbendline}d;' Jenkinsfile
+                                git add .
+                                git commit -m "stripped database lines"
+                            """
+                        }
+                    
+                        sh """#!/bin/bash
+
+                            usernameCredsId=\$(uuidgen)
+
+                            CRUMB=\$(curl -c cookies.txt 'http://${env.JENKINS_USR}:${env.JENKINS_PSW}@alloces.lan:8080/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
+                            echo \$CRUMB
+                            curl -X POST 'http://${env.JENKINS_USR}:${env.JENKINS_PSW}@alloces.lan:8080/job/gitea.arg.rip/job/${servicename}/credentials/store/folder/domain/_/createCredentials' \
+                            --data-urlencode 'json={
+                              "": "0",
+                              "credentials": {
+                                "scope": "GLOBAL",
+                                "id": "'"\$usernameCredsId"'",
+                                "username": "${servicename}",
+                                "password": "${env.pw_linuxserviceaccount}",
+                                "description": "service account login",
+                                "\$class": "com.cloudbees.plugins.credentials.impl.UsernamePasswordCredentialsImpl"
+                              }
+                            }'
+
+                            certCredsId=\$(uuidgen)
+
+                            # you git cloned ${servicename}. that's why it "already exists".
+                            ssh-keygen -t ed25519 -f "${servicename}-ssh" -N ""
+                            #chmod 600 \"${servicename}-ssh\"
+                            #chmod 600 \"${servicename}-ssh.pub\"
+                            privatekeycontent=\$(cat ${servicename}-ssh)
+                            pubkeycontent=\$(cat ${servicename}-ssh.pub)
+                            CRUMB=\$(curl -s 'http://${env.JENKINS_USR}:${env.JENKINS_PSW}@alloces.lan:8080/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
+                            echo \$CRUMB
+                            curl -X POST 'http://${env.JENKINS_USR}:${env.JENKINS_PSW}@alloces.lan:8080/job/gitea.arg.rip/job/${servicename}/credentials/store/folder/domain/_/createCredentials' \
+                            --data-urlencode 'json={
+                              "": "0",
+                              "credentials": {
+                                "scope": "GLOBAL",
+                                "id": "'"\$usernameCredsId"'",
+                                "username": "${servicename}",
+                                "password": "",
+                                "privateKeySource": {
+                                    "stapler-class": "com.cloudbees.jenkins.plugins.sshcredentials.impl.BasicSSHUserPrivateKey\$DirectEntryPrivateKeySource",
+                                      "privateKey": "\$privatekeycontent",
+                                    },
+                                    "description": "${servicename}",
+                                    "stapler-class": "com.cloudbees.jenkins.plugins.sshcredentials.impl.BasicSSHUserPrivateKey"
+                                },
+                                "description": "service account ssh",
+                                "\$class": "com.cloudbees.jenkins.plugins.sshcredentials.impl.BasicSSHUserPrivateKey"
+                              }
+                            }'
+                            privatekeycontent=
+
+                            sed -i 's/linuxServiceAccount=creds/linuxServiceAccount=credentials('${env.usernameCredsId}')/' Jenkinsfile
+                            sed -i 's/targetHost=string/targetHost="${targetHost}"/' Jenkinsfile
+
+                            rm cookies.txt
+                            pushd ${servicename}
+                            git -c core.sshCommand="ssh -i '${PK}'\" push
+                        """
+                    }
+                }
             }
         }
         stage("service account"){
             steps{
                 script {
-                    //jenkins, the user trying to SSH, must be able to ssh in and sudo
-                    ssh user@host username=$servicename password=${env.pw_linuxserviceaccount} 'echo "$ALLOCES_PSW" | sudo -Sv && bash -s' << 'ENDSSH'
-                        useradd -m -s /bin/bash $username
-                        echo "$username:$password" | chpasswd
-                        loginctl enable-linger $username
-                    ENDSSH
+                    withCredentials([sshUserPrivateKey(credentialsId: env.SUDOERSSHID, keyFileVariable: 'PK')])
+                    {
+                        sh """#!/bin/bash
+                            scp -i \"${PK}\" ${servicename}-ssh.pub ${SUDOER_USR}@${targetHost}:~/ssh.pub
+                        """
+
+                        sh """#!/bin/bash
+                            ssh-keyscan -t ed25519 ${targetHost} >> ~/.ssh/known_hosts
+                        
+                            curl -u '${env.GITEA_USR}:${env.GITEATOKEN}' https://gitea.arg.rip/greyn/deployment/raw/branch/main/scripts/serviceaccount.sh --output serviceaccount.sh
+                            sed -i 's/USERNAMETOADD=/USERNAMETOADD="${servicename}"/' serviceaccount.sh
+                            sed -i 's/PASSWORDTOADD=/PASSWORDTOADD="${env.pw_linuxserviceaccount}"/' serviceaccount.sh
+                            sed -i 's/SUDOER_PSW=/SUDOER_PSW="${SUDOER_PSW}"/' serviceaccount.sh
+
+                            ssh -i \"${PK}\" -tt ${SUDOER_USR}@${targetHost} <serviceaccount.sh
+                            rm serviceaccount.sh
+                        """
+                    }
                 }
             }
         }
@@ -59,32 +240,43 @@ pipeline {
                 //i'm pretty sure "update" with nothing will init? 
                 //meaning we don't have to init, first update will init
                 script {
-                    ssh user@host servicename=$servicename pw_productiondatabase=${env.pw_productiondatabase} pw_developmentdatabase=${env.pw_developmentdatabase} 'echo "$ALLOCES_PSW" | sudo -Sv && bash -s' << 'ENDSSH'
-                        sudo -u postgres psql && bash -s << 'ENDPSQL'
-                            create database $servicename;
-                            create user $servicename with encrypted password '$pw_productiondatabase';
-                            grant all privileges on database $servicename to $servicename;
-                        ENDPSQL
+                    withCredentials([sshUserPrivateKey(credentialsId: env.SUDOERSSHID, keyFileVariable: 'PK')])
+                    {
+                        sh """#!/bin/bash
 
-                        service_dev="${servicename}_dev"
-                        sudo -u postgres psql && bash -s << 'ENDPSQL'
-                            create database $service_dev;
-                            create user $service_dev with encrypted password '$pw_developmentdatabase';
-                            grant all privileges on database $service_dev to $service_dev;
-                        ENDPSQL
+                            curl -u '${env.GITEA_USR}:${env.GITEATOKEN}' https://gitea.arg.rip/greyn/deployment/raw/branch/main/scripts/databases.sh --output databases.sh
 
-                    ENDSSH
+                            sed -i 's/SUDOER_PSW=/SUDOER_PSW="${SUDOER_PSW}"/' databases.sh
+                            sed -i 's/pw_productiondatabase=/pw_productiondatabase="${env.pw_productiondatabase}/' databases.sh
+                            sed -i 's/pw_developmentdatabase=/pw_developmentdatabase="${env.pw_developmentdatabase}/' databases.sh
+                            sed -i 's/servicename=/servicename="${servicename}"/' databases.sh
+
+                            ssh -i \"${PK}\" -tt ${SUDOER_USR}@${targetHost} <databases.sh
+                            rm databases.sh
+                        """
+                    }
                 }
             }
         }
         stage("initial service setup"){
             steps{
-                sh 'scp $servicename.service user@server:~/.config/systemd/user/$servicename.service'
-                ssh user@host servicename=$servicename svcpw=$ARG2 'echo "$ALLOCES_PSW" | sudo -Sv && bash -s' << 'ENDSSH'
-                    systemctl --user daemon-reload
-                    systemctl --user enable $servicename.service
-                ENDSSH
+                sh """#!/bin/bash
+                    ssh -i "${servicename}-ssh" -tt ${servicename}@${targetHost} "mkdir -p ~/.config/systemd/user/"
+                    scp -i "${servicename}-ssh" ${servicename}/${servicename}.service ${servicename}@${targetHost}:~/.config/systemd/user/${servicename}.service
+
+                    ssh -i "${servicename}-ssh" -tt ${servicename}@${targetHost} 'systemctl --user daemon-reload'
+                    ssh -i "${servicename}-ssh" -tt ${servicename}@${targetHost} 'systemctl --user enable ${servicename}.service'
+                """
             }
         }
     }
+    post { 
+        failure {
+            matrixSendMessage https:true, hostname: 'greyn.club', port:8448, accessTokenCredentialsId: '040b63d1-2f14-4692-badb-114bddd7c5a5', roomId: '!QmOCACetHdGDlNFsZP:greyn.club', body: '1-click service failed :(', formattedBody: "1-click service <b>failed</b> :(" 
+        }
+        success {
+            matrixSendMessage https:true, hostname: 'greyn.club', port:8448, accessTokenCredentialsId: '040b63d1-2f14-4692-badb-114bddd7c5a5', roomId: '!QmOCACetHdGDlNFsZP:greyn.club', body: '1-click service success! go pick up the credentials!', formattedBody: '1-click service success! go pick up the credentials!'
+            //TODO: archiveArtifacts the password data, then store them somewhere
+        }
+    }
 }

scripts/databases.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+SUDOER_PSW=
+function restofscript(){
+    pw_productiondatabase=
+    pw_developmentdatabase=
+    servicename=
+    service_dev="${servicename}_dev"
+    sudo -u postgres psql -c "create database $servicename;"
+    sudo -u postgres psql -c "create user $servicename with encrypted password '$pw_productiondatabase';"
+    sudo -u postgres psql -c "grant all privileges on database $servicename to $servicename;"
+
+    sudo -u postgres psql -c "create database $service_dev;"
+    sudo -u postgres psql -c "create user $service_dev with encrypted password '$pw_developmentdatabase';"
+    sudo -u postgres psql -c "grant all privileges on database $service_dev to $service_dev;"
+
+}
+
+echo "${SUDOER_PSW}" | sudo -S bash -c "$(declare -f restofscript); restofscript"
+exit

scripts/serviceaccount.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+SUDOER_PSW=
+
+function restofscript(){
+    USERNAMETOADD=
+    PASSWORDTOADD=
+    useradd -m -s /bin/bash ${USERNAMETOADD}
+    echo "${USERNAMETOADD}:${PASSWORDTOADD}" | chpasswd
+    loginctl enable-linger ${USERNAMETOADD}
+    mkdir /home/${USERNAMETOADD}/.ssh
+    touch /home/${USERNAMETOADD}/authorized_keys
+    cat ssh.pub >> /home/${USERNAMETOADD}/.ssh/authorized_keys
+    chmod 600 /home/${USERNAMETOADD}/authorized_keys
+    chown -R "${USERNAMETOADD}:${USERNAMETOADD}" /home/${USERNAMETOADD}/.ssh
+}
+
+echo "${SUDOER_PSW}" | sudo -S bash -c "$(declare -f restofscript); restofscript"
+exit