From 24ff9549043c14dd1a4284b2665cb3902511a0d5 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 20 Jun 2024 21:41:25 +0500 Subject: [PATCH] initial commit --- Makefile | 16 +++++++ main.go | 17 +++++++ runner/runner.go | 89 ++++++++++++++++++++++++++++++++++++ runner/runner_test.go | 32 +++++++++++++ starter/cgroup_prepare.c | 51 +++++++++++++++++++++ starter/create_rootfs.sh | 17 +++++++ starter/ns_exec.c | 77 +++++++++++++++++++++++++++++++ starter/starter.c | 98 ++++++++++++++++++++++++++++++++++++++++ starter/util.h | 22 +++++++++ 9 files changed, 419 insertions(+) create mode 100644 Makefile create mode 100644 main.go create mode 100644 runner/runner.go create mode 100644 runner/runner_test.go create mode 100644 starter/cgroup_prepare.c create mode 100755 starter/create_rootfs.sh create mode 100644 starter/ns_exec.c create mode 100644 starter/starter.c create mode 100644 starter/util.h diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..87bde14 --- /dev/null +++ b/Makefile @@ -0,0 +1,16 @@ +all: + rm -rf starter/alpine-make-rootfs + git clone https://github.com/alpinelinux/alpine-make-rootfs + mv alpine-make-rootfs starter/alpine-make-rootfs + go build -o ./run + gcc starter/starter.c -o starter/starter + starter/create_rootfs.sh +install_minrootfs: + rm -rf starter/alpine-make-rootfs + git clone https://github.com/alpinelinux/alpine-make-rootfs + mv alpine-make-rootfs starter/alpine-make-rootfs +mkrootfs: + starter/create_rootfs.sh +compile: + go build -o ./run + gcc starter/starter.c -o starter/starter diff --git a/main.go b/main.go new file mode 100644 index 0000000..360178a --- /dev/null +++ b/main.go @@ -0,0 +1,17 @@ +package main + +import ( + //"os" + //"fmt" + //"log" + //exec "os/exec" + runner "runner/runner" +) + +func main() { + err:=runner.Init() + if(err!=nil) { + panic(err) + } + //runner.IsolatedRun(exec.Command("ls", "/")) +} diff --git a/runner/runner.go b/runner/runner.go new file mode 100644 index 0000000..1d87c62 --- /dev/null +++ b/runner/runner.go @@ -0,0 +1,89 @@ +package runner + +import ( + "errors" + "fmt" + "log" + "os" + "strings" + exec "os/exec" + cgroups "github.com/containerd/cgroups" + //cgroup2 "github.com/containerd/cgroups/v3/cgroup2" + //specs "github.com/opencontainers/runtime-spec/specs-go" + rand "math/rand" + "time" + "os/user" +) + +const runner_username string = "gaterunner" +const runIdLength = 20 +var coresIsolated []int + +type Limits { + core int + memory int +} + +func extractNumbers(s string) (result []int) { + lastNumber,isNumber := false,false + var curNumber=0 + for _,char :=range s { + isNumber=(char>='0' && char<='9') + if(isNumber) { + curNumber*=10 + curNumber+=int(char-'0') + } + if(!isNumber && lastNumber) { + result=append(result,curNumber) + curNumber=0 + } + lastNumber=isNumber + } + if(lastNumber) { + result=append(result,curNumber) + } + return +} + +func Init() error { + rand.Seed(time.Now().UnixNano()) + //croup initialisation: + if cgroups.Mode() == cgroups.Unified { + log.Println("cgroups v2 usage approved") + } else { + return fmt.Errorf("cgroups v2 are not enabled")//TODO: trouble description + } + //isolated cores initialisation: + cmdlineBytes := make([]byte, 400) + cmdlineFile,_:=os.Open("/proc/cmdline"); + countCmdlineBytes,_:=cmdlineFile.Read(cmdlineBytes); + cmdline:=string(cmdlineBytes[:countCmdlineBytes]) + kernelParams:=strings.Split(cmdline," ") + for _,param := range kernelParams{ + if(len(param)>=9 && param[:9]=="isolcpus=") { + coresIsolated=append(coresIsolated, extractNumbers(param[9:])...) + } + } + if(len(coresIsolated)==0) { + return fmt.Errorf("no free cores available")//TODO: trouble description + } + log.Println("running on cores:",coresIsolated) + //user setup: + _,err:=user.Lookup(runner_username) + if err!=nil{ + if errors.As(err, new(user.UnknownUserError)) { + exec.Command("useradd", runner_username).Run(); + } else { + return fmt.Errorf("user error:",err) + } + } + log.Println("Runner initialisation successful!") + return nil +} + +func IsolatedRun(command []string,core int) { + var runId string + for i:=0;i +#include +#include +#include +#include + +#define CGROUP_NAME_SIZE 20 +char cgroup_name[CGROUP_NAME_SIZE+1]; +const char* cgroups_path="/sys/fs/cgroup/"; + +void prepare_cgroup(struct limits* limits) { + for(int i=0;imemory); + write_file("memory.max",memory_string); + free(memory_string); + char* cpus_string=NULL; + asprintf(&cpus_string,"%d\n",limits->core); + write_file("cpuset.cpus",cpus_string); + free(cpus_string); + write_file("pids.max","1\n"); + //write_file("cpuset.cpus","3\n"); + chdir(cwd); +} + +void add_to_cgroup(int pid) { + char cwd[PATH_MAX]; + if(getcwd(cwd,sizeof(cwd))==NULL) die("getcwd error: %m"); + chdir(cgroups_path); + chdir(cgroup_name); + char* pidstr=NULL; + asprintf(&pidstr,"%d\n",pid); + write_file("cgroup.procs",pidstr); + free(pidstr); + chdir(cwd); +} + +void remove_cgroup() { + char cwd[PATH_MAX]; + if(getcwd(cwd,sizeof(cwd))==NULL) die("getcwd error: %m"); + if(chdir(cgroups_path)) die("chdir error: %m"); + if(rmdir(cgroup_name)) die("rmdir error: %m"); + if(chdir(cwd)) die("chdir error: %m"); +} diff --git a/starter/create_rootfs.sh b/starter/create_rootfs.sh new file mode 100755 index 0000000..ea1349b --- /dev/null +++ b/starter/create_rootfs.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +if [[ $(whoami) != root ]] +then + echo "run as root please" + exit +fi + +NAME=minrootfs +SCRIPT_DIR=$(dirname -- "$(readlink -f "${BASH_SOURCE}")") + +rm -rf $SCRIPT_DIR/$NAME +sudo $SCRIPT_DIR/alpine-make-rootfs/alpine-make-rootfs $SCRIPT_DIR/$NAME +#chown -R nobody:nogroup $SCRIPT_DIR/$NAME +#rm -r ../$NAME +#mv $NAME ../$NAME + diff --git a/starter/ns_exec.c b/starter/ns_exec.c new file mode 100644 index 0000000..13ff4ef --- /dev/null +++ b/starter/ns_exec.c @@ -0,0 +1,77 @@ +#include +#include +#include + +static void procfs_prepare() +{ + if (mkdir("/proc", 0555) && errno != EEXIST) die("Failed to mkdir /proc: %m\n"); + if (mount("proc", "/proc", "proc", 0, "")) die("Failed to mount proc: %m\n"); +} + +#define hostname "runner" + +#define put_old "oldfs" +//#define shared_folder "../shared" +//#define shared_mountpoint "shared" +void mnt_prepare(char* rootfs, char* shared_mountpoint) { + //char* shared_folder = malloc(strlen(shared_mountpoint)+4); + //strcpy(shared_folder,"../"); + //strcat(shared_folder,shared_mountpoint); + char* shared_folder=NULL; + asprintf(&shared_folder,"../%s",shared_mountpoint); + if (mount(rootfs,rootfs,"ext4",MS_BIND,"")) die("failed to mount %s: %m", rootfs); + //if (mount(shared_mountpoint,shared_mountpoint,"ext4",MS_BIND,"")) die("failed to mount %s: %m",shared_mountpoint); + if (chdir(rootfs)) die("falied to cd:%m"); + if (mount("/sys","sys","sysfs",0,"")) die("failed to mount: %m"); + //if (mount("/dev","dev","udev",0,"")) die("failed to mount: %m"); + if (mkdir("shared", 0777) && errno != EEXIST) die("Failed to mkdir %s: %m\n", shared_mountpoint); + if (mount(shared_folder,"shared","ext4",MS_BIND,"")) die("failed to mount: %m"); + if (mkdir(put_old, 0000) && errno != EEXIST) die("Failed to mkdir %s: %m\n", put_old); + if (syscall(SYS_pivot_root, ".", put_old)) die("Failed to pivot_root from %s to %s: %m\n", rootfs, put_old); + if (chdir("/")) die("Failed to chdir to new root: %m\n"); + procfs_prepare(); + if (umount2(put_old, MNT_DETACH)) die("Failed to umount put_old %s: %m\n", put_old); + if (rmdir(put_old)) die("Failed to rmdir: %m"); + free(shared_folder); +} +void ro_fs(char* shared_mountpoint) { + char* shared_folder = malloc(strlen(shared_mountpoint)+4); + strcpy(shared_folder,"../"); + strcat(shared_folder,shared_mountpoint); + if (mount("/","/","ext4",MS_REMOUNT | MS_RDONLY | MS_BIND,"")) die("failed to mount: %m"); + if (mount(shared_mountpoint,"shared","ext4",MS_REMOUNT | MS_RDONLY | MS_BIND,"")) die("failed to mount: %m"); + free(shared_folder); +} +//#undef shared_mountpoint +//#undef shared_folder +#undef put_old + +void await_setup(int pipe) { // wait for signal from parent + char buf[2]; + if (read(pipe, buf, 2) != 2) die("Failed to read from pipe: %m\n"); +} + +static int nsrun(void* arg) { +#define setup "./setup.sh" + //die when parent dies + if (prctl(PR_SET_PDEATHSIG, SIGKILL)) die("cannot PR_SET_PDEATHSIG for child process: %m\n"); + struct params *params = (struct params*) arg; + await_setup(params->fd[0]); + mnt_prepare("minrootfs",params->shared_folder); + //if(chdir("shared")) die("faled to chdir: %m"); + sethostname(hostname,sizeof(hostname)); + //if(chdir("..")) die("faled to chdir: %m"); + ro_fs(params->shared_folder); + if(setgid(1000)) die("failed to setgid:%m"); + if(setuid(1000)) die("failed to setuid:%m"); + char** argv = params->argv; + char* cmd = argv[0]; + char* env[]={ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "PWD=/", + "NULL", + }; + if (execve(cmd,argv,env) == -1) die("failed to exec,%m"); + return 1; +#undef setup +} diff --git a/starter/starter.c b/starter/starter.c new file mode 100644 index 0000000..320e2d9 --- /dev/null +++ b/starter/starter.c @@ -0,0 +1,98 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "util.h" + +struct params { + char* shared_folder; + int fd[2]; + char **argv; +}; + +struct limits { + size_t memory; + int core; +}; + +#include "ns_exec.c" +#include "cgroup_prepare.c" + +//const size_t STACK_SIZE=1000000; +#define STACK_SIZE 1000000 +static char nmstack[STACK_SIZE]; + +static void parse_args(int argc, char **argv, struct params *params,struct limits *limits){ + if (argc < 5) { + printf("usage:\n starter "); + exit(0); + } + argc--;argv++; + limits->core=atoi(argv[0]); + argc--;argv++; + limits->memory=atoi(argv[0]); + argc--;argv++; + params->shared_folder=argv[0]; + argc--;argv++; + + params->argv = argv; +} + +static void prepare_userns(int pid) { + char path[100]; + char line[100]; + + int uid = getuid(); + int gid = getgid(); + //int unprivileged_uid=1002; + //int unprivileged_gid=1002; + int unprivileged_uid=66534; + int unprivileged_gid=65534; + + sprintf(path, "/proc/%d/uid_map", pid); + sprintf(line, "0 %d 1\n1 %d 1000\n", uid, unprivileged_uid); + write_file(path, line); + + sprintf(path, "/proc/%d/setgroups", pid); + sprintf(line, "deny"); + write_file(path, line); + + sprintf(path, "/proc/%d/gid_map", pid); + sprintf(line, "0 %d 1\n1 %d 1000\n", gid, unprivileged_gid); + write_file(path, line); +} + + +int main(int argc,char** argv) { + srand(time(NULL)); + struct params params; + memset(¶ms, 0, sizeof(struct params)); + struct limits limits; + memset(&limits, 0, sizeof(struct limits)); + parse_args(argc, argv, ¶ms, &limits); + prepare_cgroup(&limits); + //exit(0); + if(setuid(0)) die("need to be run as root"); + if(setgid(0)) die("need to be run as root"); + if (pipe(params.fd) < 0) exit(0); + int clone_flags = SIGCHLD | CLONE_NEWUTS | CLONE_NEWUSER | CLONE_NEWNS | CLONE_NEWPID | CLONE_NEWNET | CLONE_NEWIPC | CLONE_NEWCGROUP/* | CLONE_NEWTIME*/; + int nsrun_pid=clone(nsrun,nmstack+STACK_SIZE,clone_flags,¶ms); + int pipe=params.fd[1]; + //sleep(1); + prepare_userns(nsrun_pid); + if (nsrun_pid<0) {die("faled to clone");} + add_to_cgroup(nsrun_pid); + if (write(pipe, "OK", 2) != 2) die("Failed to write to pipe: %m");//report readiness + if (waitpid(nsrun_pid, NULL, 0) == -1) die("Failed to wait pid %d: %m\n", nsrun_pid); + //int pipe=params.fd[1]; + remove_cgroup(); + return 0; +} diff --git a/starter/util.h b/starter/util.h new file mode 100644 index 0000000..4f1484f --- /dev/null +++ b/starter/util.h @@ -0,0 +1,22 @@ +//#ifndef ISOLATE_UTIL_H +//#define ISOLATE_UTIL_H + +static void die(const char *fmt, ...) +{ + va_list params; + + va_start(params, fmt); + vfprintf(stderr, fmt, params); + va_end(params); + exit(1); +} + +static void write_file(char* path, char* line) +{ + FILE *f = fopen(path, "w"); + if (f == NULL) {die("Failed to open file %s: %m\n", path);} + if (fwrite(line, 1, strlen(line), f) < 0) {die("Failed to write to file %s:\n", path);} + if (fclose(f) != 0) {die("Failed to close file %s: %m\n", path);} +} + +//#endif //ISOLATE_UTIL_H