clore
/
hosting
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use wrapped cli to create containers

This commit is contained in:
clore 2024-04-01 01:00:32 +00:00
parent b4a1721213
commit ef779233b9
3 changed files with 129 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lib/config.py
View File

@ -14,7 +14,7 @@ hard_config = {
"run_iptables_with_sudo":True,
"clore_iptables_rules":[
"-A INPUT -s <subnet> -j DROP",
"-I DOCKER-USER -i <interface> -d <subnet> -j DROP"
"-I FORWARD -i <interface> -d <subnet> -j DROP"
],
"clore_br_first_allowed_octet":"172",
"ws_peers_recheck_interval": 300,
@ -31,7 +31,8 @@ hard_config = {
"max_container_log_size": 262144, # Characters
"container_log_streaming_interval": 2, # Seconds
"maximum_service_loop_time": 900, # Seconds, failsafe variable - if service is stuck processing longer than this timeframe it will lead into restarting the app
"maximum_pull_service_loop_time": 14400 # Exception for image pulling
"maximum_pull_service_loop_time": 14400, # Exception for image pulling
"creation_engine": "wrapper" # "wrapper" or "sdk" | Wrapper - wrapped docker cli, SDK - docker sdk
}
parser = argparse.ArgumentParser(description='Example argparse usage')

112
lib/docker_cli_wrapper.py Normal file
View File

@ -0,0 +1,112 @@
from lib import logging as logging_lib
from lib import config as config_module
import subprocess
import re, os
import signal
import docker
config = config_module.config
log = logging_lib.log
def create_container(container_options, ip=None, docker_gpus=False, timeout=30):
# Sanitize and validate input
container_options = sanitize_input(container_options)
command = ["docker", "run", "--detach", "--tty"]
if "name" in container_options:
command.extend(["--name", container_options["name"]])
if "network_mode" in container_options:
command.extend(["--network", container_options["network_mode"]])
if "cap_add" in container_options:
for cap in container_options["cap_add"]:
command.extend(["--cap-add", cap])
if "volumes" in container_options:
for volume_host, volume_container in container_options["volumes"].items():
bind = f"{volume_host}:{volume_container['bind']}"
if "mode" in volume_container:
bind += f":{volume_container['mode']}"
command.extend(["--volume", bind])
if "ports" in container_options:
for port_container, port_host in container_options["ports"].items():
command.extend(["-p", f"{port_host}:{port_container}"])
if "environment" in container_options:
for env_var, env_value in container_options["environment"].items():
command.extend(["-e", f"{env_var}={env_value}"])
if "log_config" in container_options:
log_config = container_options["log_config"]
if isinstance(log_config, dict):
for key, value in log_config["Config"].items():
log_option = (f"{key}={value}")
command.extend(["--log-opt", log_option])
command.extend(["--log-driver", log_config["Type"]])
else:
log.debug("Invalid log_config format. Skipping log configuration.")
if "runtime" in container_options:
command.extend(["--runtime", container_options["runtime"]])
if docker_gpus:
if type(docker_gpus)==list:
command.extend(['--gpus', '"device=' + ','.join(str(gpu_id) for gpu_id in docker_gpus) + '"'])
else:
command.extend(["--gpus", "all"])
if "storage_opt" in container_options:
for storage_opt, value in container_options["storage_opt"].items():
command.extend(["--storage-opt", f"{storage_opt}={value}"])
if "entrypoint" in container_options:
command.extend(["--entrypoint", container_options["entrypoint"]])
if ip:
command.extend(["--ip", ip])
command.append(container_options["image"])
try:
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
try:
output, error = process.communicate(timeout=timeout)
if process.returncode == 0:
container_id = output.strip()
return container_id
else:
print(f"Error creating container: {error}")
return None
except subprocess.TimeoutExpired:
process.send_signal(signal.SIGTERM)
process.kill()
print("Timeout exceeded while creating container.")
return None
except subprocess.CalledProcessError as e:
print(f"Error creating container: {e.output}")
return None
def sanitize_input(container_options):
sanitized_options = {}
for key, value in container_options.items():
if isinstance(value, str):
# Remove any potential shell injection or escapes
sanitized_value = re.sub(r'[`$\\\'\"]', '', value)
sanitized_options[key] = sanitized_value
elif isinstance(value, dict):
sanitized_options[key] = sanitize_input(value)
elif isinstance(value, list):
sanitized_list = []
for item in value:
if isinstance(item, str):
sanitized_item = re.sub(r'[`$\\\'\"]', '', item)
sanitized_list.append(sanitized_item)
elif isinstance(item, dict):
sanitized_list.append(sanitize_input(item))
sanitized_options[key] = sanitized_list
else:
sanitized_options[key] = value
return sanitized_options

21
lib/docker_deploy.py
View File

@ -1,5 +1,6 @@
from lib import config as config_module
from lib import logging as logging_lib
from lib import docker_cli_wrapper
from lib import docker_interface
from lib import get_specs
import docker
@ -44,6 +45,7 @@ def deploy(validated_containers):
try:
image_ready = False
docker_gpus = None
for local_image in local_images:
if local_image.replace(':latest','')==validated_container["image"].replace(':latest',''):
image_ready = True
@ -76,14 +78,16 @@ def deploy(validated_containers):
if "network" in validated_container:
container_options["network_mode"]=validated_container["network"]
if "ip" in validated_container:
if "ip" in validated_container and config.creation_engine=="sdk":
del container_options["network_mode"]
if "gpus" in validated_container and type(validated_container["gpus"])==bool:
container_options["runtime"]="nvidia"
docker_gpus=True
container_options["device_requests"].append(docker.types.DeviceRequest(count=-1, capabilities=[['gpu']]))
elif "gpus" in validated_container and type(validated_container["gpus"])==list:
container_options["runtime"]="nvidia"
docker_gpus=validated_container["gpus"]
container_options["device_requests"].append(docker.types.DeviceRequest(
count=-1,
capabilities=[['gpu']],
@ -118,12 +122,15 @@ def deploy(validated_containers):
container_options["entrypoint"]=validated_container["entrypoint_command"]
if not validated_container["name"] in created_container_names and image_ready:
container = client.containers.create(**container_options)
if "ip" in validated_container:
client.networks.get(validated_container["network"] if "network" in validated_container else "clore-br0").connect(container, ipv4_address=validated_container["ip"])
client.networks.get("bridge").disconnect(container)
if not "paused" in validated_container:
container.start()
if config.creation_engine == "wrapper":
docker_cli_wrapper.create_container(container_options, ip=(validated_container["ip"] if "ip" in validated_container else None), docker_gpus=docker_gpus)
else:
container = client.containers.create(**container_options)
if "ip" in validated_container:
client.networks.get(validated_container["network"] if "network" in validated_container else "clore-br0").connect(container, ipv4_address=validated_container["ip"])
client.networks.get("bridge").disconnect(container)
if not "paused" in validated_container:
container.start()
except Exception as e:
log.debug(f"Container creation issue | {e}")
pass