initial commit

Makefile

@@ -0,0 +1,16 @@
+all:
+	rm -rf starter/alpine-make-rootfs
+	git clone https://github.com/alpinelinux/alpine-make-rootfs
+	mv alpine-make-rootfs starter/alpine-make-rootfs
+	go build -o ./run
+	gcc starter/starter.c -o starter/starter
+	starter/create_rootfs.sh
+install_minrootfs:
+	rm -rf starter/alpine-make-rootfs
+	git clone https://github.com/alpinelinux/alpine-make-rootfs
+	mv alpine-make-rootfs starter/alpine-make-rootfs
+mkrootfs:
+	starter/create_rootfs.sh
+compile:
+	go build -o ./run
+	gcc starter/starter.c -o starter/starter

main.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+    //"os"
+    //"fmt"
+    //"log"
+    //exec "os/exec"
+    runner "runner/runner"
+)
+
+func main() {
+    err:=runner.Init()
+    if(err!=nil) {
+        panic(err)
+    }
+    //runner.IsolatedRun(exec.Command("ls", "/"))
+}

runner/runner.go

@@ -0,0 +1,89 @@
+package runner
+
+import (
+    "errors"
+    "fmt"
+    "log"
+    "os"
+    "strings"
+    exec "os/exec"
+    cgroups "github.com/containerd/cgroups"
+    //cgroup2 "github.com/containerd/cgroups/v3/cgroup2"
+    //specs "github.com/opencontainers/runtime-spec/specs-go"
+    rand "math/rand"
+    "time"
+    "os/user"
+)
+
+const runner_username string = "gaterunner"
+const runIdLength = 20
+var coresIsolated []int
+
+type Limits {
+    core    int
+    memory  int
+}
+
+func extractNumbers(s string) (result []int) {
+    lastNumber,isNumber := false,false
+    var curNumber=0
+    for _,char :=range s {
+        isNumber=(char>='0' && char<='9')
+        if(isNumber) {
+            curNumber*=10
+            curNumber+=int(char-'0')
+        }
+        if(!isNumber && lastNumber) {
+            result=append(result,curNumber)
+            curNumber=0
+        }
+        lastNumber=isNumber
+    }
+    if(lastNumber) {
+        result=append(result,curNumber)
+    }
+    return
+}
+
+func Init() error {
+    rand.Seed(time.Now().UnixNano())
+    //croup initialisation:
+    if cgroups.Mode() == cgroups.Unified {
+        log.Println("cgroups v2 usage approved")
+    } else {
+        return fmt.Errorf("cgroups v2 are not enabled")//TODO: trouble description
+    }
+    //isolated cores initialisation:
+    cmdlineBytes := make([]byte, 400)
+    cmdlineFile,_:=os.Open("/proc/cmdline");
+    countCmdlineBytes,_:=cmdlineFile.Read(cmdlineBytes);
+    cmdline:=string(cmdlineBytes[:countCmdlineBytes])
+    kernelParams:=strings.Split(cmdline," ")
+    for _,param := range kernelParams{
+        if(len(param)>=9 && param[:9]=="isolcpus=") {
+            coresIsolated=append(coresIsolated, extractNumbers(param[9:])...)
+        }
+    }
+    if(len(coresIsolated)==0) {
+        return fmt.Errorf("no free cores available")//TODO: trouble description
+    }
+    log.Println("running on cores:",coresIsolated)
+    //user setup:
+    _,err:=user.Lookup(runner_username)
+    if err!=nil{
+        if errors.As(err, new(user.UnknownUserError)) {
+            exec.Command("useradd", runner_username).Run();
+        } else {
+            return fmt.Errorf("user error:",err)
+        }
+    }
+    log.Println("Runner initialisation successful!")
+    return nil
+}
+
+func IsolatedRun(command []string,core int) {
+    var runId string
+    for i:=0;i<runIdLength;i++{
+        runId+=string('a'+byte(rand.Int31n(26)))
+    }
+}

runner/runner_test.go

@@ -0,0 +1,32 @@
+package runner
+
+import (
+    "testing"
+)
+
+type ENTestPair struct {
+    test string
+    result []int
+}
+
+var ENTests = []ENTestPair {
+    {"123",[]int{123}},
+    {"abc123abc",[]int{123}},
+    {"",[]int{}},
+    {"0",[]int{0}},
+    {"1a2a3a6",[]int{1,2,3,6}},
+}
+
+func TestExtractNumbers(t *testing.T) {
+    for _,test:=range(ENTests) {
+        result:=extractNumbers(test.test)
+        if(len(result)!=len(test.result)) {
+            t.Error("for",test.test,"expected",test.result,"got",result)
+        }
+        for i:=0;i<len(result);i++ {
+            if(result[i]!=test.result[i]) {
+            t.Error("for",test.test,"expected",test.result,"got",result)
+            }
+        }
+    }
+}

starter/cgroup_prepare.c

@@ -0,0 +1,51 @@
+#include<sys/prctl.h>
+#include<sys/mount.h>
+#include<sys/stat.h>
+#include<unistd.h>
+#include<linux/limits.h>
+
+#define CGROUP_NAME_SIZE 20
+char cgroup_name[CGROUP_NAME_SIZE+1];
+const char* cgroups_path="/sys/fs/cgroup/";
+
+void prepare_cgroup(struct limits* limits) {
+    for(int i=0;i<CGROUP_NAME_SIZE;i++) {
+        cgroup_name[i]='a'+rand()%26;
+    }
+    char cwd[PATH_MAX];
+    if(getcwd(cwd,sizeof(cwd))==NULL) die("getcwd error: %m");
+    chdir(cgroups_path);
+    mkdir(cgroup_name,0755);
+    chdir(cgroup_name);
+    char* memory_string=NULL;
+    asprintf(&memory_string,"%d\n",limits->memory);
+    write_file("memory.max",memory_string);
+    free(memory_string);
+    char* cpus_string=NULL;
+    asprintf(&cpus_string,"%d\n",limits->core);
+    write_file("cpuset.cpus",cpus_string);
+    free(cpus_string);
+    write_file("pids.max","1\n");
+    //write_file("cpuset.cpus","3\n");
+    chdir(cwd);
+}
+
+void add_to_cgroup(int pid) {
+    char cwd[PATH_MAX];
+    if(getcwd(cwd,sizeof(cwd))==NULL) die("getcwd error: %m");
+    chdir(cgroups_path);
+    chdir(cgroup_name);
+    char* pidstr=NULL;
+    asprintf(&pidstr,"%d\n",pid);
+    write_file("cgroup.procs",pidstr);
+    free(pidstr);
+    chdir(cwd);
+}
+
+void remove_cgroup() {
+    char cwd[PATH_MAX];
+    if(getcwd(cwd,sizeof(cwd))==NULL)   die("getcwd error: %m");
+    if(chdir(cgroups_path))             die("chdir error: %m");
+    if(rmdir(cgroup_name))              die("rmdir error: %m");
+    if(chdir(cwd))                      die("chdir error: %m");
+}

starter/create_rootfs.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [[ $(whoami) != root ]]
+then
+    echo "run as root please"
+    exit
+fi
+
+NAME=minrootfs
+SCRIPT_DIR=$(dirname -- "$(readlink -f "${BASH_SOURCE}")")
+
+rm -rf $SCRIPT_DIR/$NAME
+sudo $SCRIPT_DIR/alpine-make-rootfs/alpine-make-rootfs $SCRIPT_DIR/$NAME
+#chown -R nobody:nogroup $SCRIPT_DIR/$NAME
+#rm -r ../$NAME
+#mv $NAME ../$NAME
+

starter/ns_exec.c

@@ -0,0 +1,77 @@
+#include<sys/prctl.h>
+#include<sys/mount.h>
+#include<sys/stat.h>
+
+static void procfs_prepare()
+{
+    if (mkdir("/proc", 0555) && errno != EEXIST)                                    die("Failed to mkdir /proc: %m\n");
+    if (mount("proc", "/proc", "proc", 0, ""))                                      die("Failed to mount proc: %m\n");
+}
+
+#define hostname "runner"
+
+#define put_old "oldfs"
+//#define shared_folder "../shared"
+//#define shared_mountpoint "shared"
+void mnt_prepare(char* rootfs, char* shared_mountpoint) {
+    //char* shared_folder = malloc(strlen(shared_mountpoint)+4);
+    //strcpy(shared_folder,"../");
+    //strcat(shared_folder,shared_mountpoint);
+    char* shared_folder=NULL;
+    asprintf(&shared_folder,"../%s",shared_mountpoint);
+    if (mount(rootfs,rootfs,"ext4",MS_BIND,""))                                     die("failed to mount %s: %m", rootfs);
+    //if (mount(shared_mountpoint,shared_mountpoint,"ext4",MS_BIND,""))                   die("failed to mount %s: %m",shared_mountpoint);
+    if (chdir(rootfs))                                                              die("falied to cd:%m");
+    if (mount("/sys","sys","sysfs",0,""))                                      die("failed to mount: %m");
+    //if (mount("/dev","dev","udev",0,""))                                      die("failed to mount: %m");
+    if (mkdir("shared", 0777) && errno != EEXIST)                          die("Failed to mkdir %s: %m\n", shared_mountpoint);
+    if (mount(shared_folder,"shared","ext4",MS_BIND,""))                   die("failed to mount: %m");
+    if (mkdir(put_old, 0000) && errno != EEXIST)                                    die("Failed to mkdir %s: %m\n", put_old);
+    if (syscall(SYS_pivot_root, ".", put_old))                                      die("Failed to pivot_root from %s to %s: %m\n", rootfs, put_old);
+    if (chdir("/"))                                                                 die("Failed to chdir to new root: %m\n");
+    procfs_prepare();
+    if (umount2(put_old, MNT_DETACH))                                               die("Failed to umount put_old %s: %m\n", put_old);
+    if (rmdir(put_old))                                                             die("Failed to rmdir: %m");
+    free(shared_folder);
+}
+void ro_fs(char* shared_mountpoint) {
+    char* shared_folder = malloc(strlen(shared_mountpoint)+4);
+    strcpy(shared_folder,"../");
+    strcat(shared_folder,shared_mountpoint);
+    if (mount("/","/","ext4",MS_REMOUNT | MS_RDONLY | MS_BIND,""))                  die("failed to mount: %m");
+    if (mount(shared_mountpoint,"shared","ext4",MS_REMOUNT | MS_RDONLY | MS_BIND,""))  die("failed to mount: %m");
+    free(shared_folder);
+}
+//#undef shared_mountpoint
+//#undef shared_folder
+#undef put_old
+
+void await_setup(int pipe) { // wait for signal from parent
+    char buf[2];
+    if (read(pipe, buf, 2) != 2) die("Failed to read from pipe: %m\n");
+}
+
+static int nsrun(void* arg) {
+#define setup "./setup.sh"
+    //die when parent dies
+    if (prctl(PR_SET_PDEATHSIG, SIGKILL)) die("cannot PR_SET_PDEATHSIG for child process: %m\n");
+    struct params *params = (struct params*) arg;
+    await_setup(params->fd[0]);
+    mnt_prepare("minrootfs",params->shared_folder);
+    //if(chdir("shared")) die("faled to chdir: %m");
+    sethostname(hostname,sizeof(hostname));
+    //if(chdir("..")) die("faled to chdir: %m");
+    ro_fs(params->shared_folder);
+    if(setgid(1000)) die("failed to setgid:%m");
+    if(setuid(1000)) die("failed to setuid:%m");
+    char** argv = params->argv;
+    char* cmd = argv[0];
+    char* env[]={
+        "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+        "PWD=/",
+        "NULL",
+    };
+    if (execve(cmd,argv,env) == -1) die("failed to exec,%m");
+    return 1;
+#undef setup
+}

starter/starter.c

@@ -0,0 +1,98 @@
+#define _GNU_SOURCE
+#include <sys/wait.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+#include <errno.h>
+#include <sched.h>
+#include <sys/syscall.h>
+#include <linux/sched.h>
+#include <stdarg.h>
+#include <time.h>
+#include "util.h"
+
+struct params {
+    char* shared_folder;
+    int fd[2];
+    char **argv;
+};
+
+struct limits {
+    size_t memory;
+    int core;
+};
+
+#include "ns_exec.c"
+#include "cgroup_prepare.c"
+
+//const size_t STACK_SIZE=1000000;
+#define STACK_SIZE 1000000
+static char nmstack[STACK_SIZE];
+
+static void parse_args(int argc, char **argv, struct params *params,struct limits *limits){
+    if (argc < 5) {
+        printf("usage:\n starter <core_id> <memory_amount> <shared folder> <command, arg1,arg2,...>");
+        exit(0);
+    }
+    argc--;argv++;
+    limits->core=atoi(argv[0]);
+    argc--;argv++;
+    limits->memory=atoi(argv[0]);
+    argc--;argv++;
+    params->shared_folder=argv[0];
+    argc--;argv++;
+
+    params->argv = argv;
+}
+
+static void prepare_userns(int pid) {
+    char path[100];
+    char line[100];
+
+    int uid = getuid();
+    int gid = getgid();
+    //int unprivileged_uid=1002;
+    //int unprivileged_gid=1002;
+    int unprivileged_uid=66534;
+    int unprivileged_gid=65534;
+
+    sprintf(path, "/proc/%d/uid_map", pid);
+    sprintf(line, "0 %d 1\n1 %d 1000\n", uid, unprivileged_uid);
+    write_file(path, line);
+
+    sprintf(path, "/proc/%d/setgroups", pid);
+    sprintf(line, "deny");
+    write_file(path, line);
+
+    sprintf(path, "/proc/%d/gid_map", pid);
+    sprintf(line, "0 %d 1\n1 %d 1000\n", gid, unprivileged_gid);
+    write_file(path, line);
+}
+
+
+int main(int argc,char** argv) {
+    srand(time(NULL));
+    struct params params;
+    memset(&params, 0, sizeof(struct params));
+    struct limits limits;
+    memset(&limits, 0, sizeof(struct limits));
+    parse_args(argc, argv, &params, &limits);
+    prepare_cgroup(&limits);
+    //exit(0);
+    if(setuid(0)) die("need to be run as root");
+    if(setgid(0)) die("need to be run as root");
+    if (pipe(params.fd) < 0) exit(0);
+    int clone_flags = SIGCHLD | CLONE_NEWUTS | CLONE_NEWUSER | CLONE_NEWNS | CLONE_NEWPID | CLONE_NEWNET | CLONE_NEWIPC | CLONE_NEWCGROUP/* | CLONE_NEWTIME*/;
+    int nsrun_pid=clone(nsrun,nmstack+STACK_SIZE,clone_flags,&params);
+    int pipe=params.fd[1];
+    //sleep(1);
+    prepare_userns(nsrun_pid);
+    if (nsrun_pid<0) {die("faled to clone");}
+    add_to_cgroup(nsrun_pid);
+    if (write(pipe, "OK", 2) != 2) die("Failed to write to pipe: %m");//report readiness
+    if (waitpid(nsrun_pid, NULL, 0) == -1) die("Failed to wait pid %d: %m\n", nsrun_pid);
+    //int pipe=params.fd[1];
+    remove_cgroup();
+    return 0;
+}

starter/util.h

@@ -0,0 +1,22 @@
+//#ifndef ISOLATE_UTIL_H
+//#define ISOLATE_UTIL_H
+
+static void die(const char *fmt, ...)
+{
+    va_list params;
+
+    va_start(params, fmt);
+    vfprintf(stderr, fmt, params);
+    va_end(params);
+    exit(1);
+}
+
+static void write_file(char* path, char* line)
+{
+    FILE *f = fopen(path, "w");
+    if (f == NULL) {die("Failed to open file %s: %m\n", path);}
+    if (fwrite(line, 1, strlen(line), f) < 0) {die("Failed to write to file %s:\n", path);}
+    if (fclose(f) != 0) {die("Failed to close file %s: %m\n", path);}
+}
+
+//#endif //ISOLATE_UTIL_H