diff --git a/clore_hosting/main.py b/clore_hosting/main.py index c447946..fef7e78 100644 --- a/clore_hosting/main.py +++ b/clore_hosting/main.py @@ -4,6 +4,7 @@ from lib import log_streaming_task from lib import run_startup_script from lib import hive_miner_interface from lib import docker_interface +from lib import background_job from lib import docker_deploy from lib import clore_partner from lib import clore_partner_socket @@ -470,6 +471,11 @@ class CloreClient: if result.success: self.last_checked_ws_peers = utils.unix_timestamp() self.allowed_images=result.allowed_images+self.extra_allowed_images + if self.xfs_state == "active": + self.allowed_images.append({ + "repository": "vastai/test", + "allowed_tags": ["bandwidth-test-nvidia"] + }) if not config.debug_ws_peer: for pure_ws_peer in result.ws_peers: self.ws_peers[pure_ws_peer]={ @@ -547,7 +553,7 @@ class CloreClient: await monitoring.put("oc_service") oc_apply_allowed = True ### OC Service should also hande Hive stuff - if self.use_hive_flightsheet and self.is_hive and not self.dont_use_hive_binaries: + if self.use_hive_flightsheet and self.is_hive and not self.dont_use_hive_binaries and background_job.is_enabled(): await set_hive_miner_status(True) oc_apply_allowed = False # Don't apply any OC when running HiveOS miner elif self.is_hive and not self.dont_use_hive_binaries: diff --git a/lib/background_job.py b/lib/background_job.py new file mode 100644 index 0000000..177d4a4 --- /dev/null +++ b/lib/background_job.py @@ -0,0 +1,22 @@ +import time +import re + +disabled_till = 0 + +def is_background_job_container_name(string): + if type(string) != str: + return False + pattern = r"^clore-default-\d+$" + return bool(re.match(pattern, string)) + +def temporarly_disable(seconds): + global disabled_till + disabled_till = time.time() + seconds + +def enable(): + global disabled_till + disabled_till=0 + +def is_enabled(): + global disabled_till + return True if disabled_till < time.time() else False \ No newline at end of file diff --git a/lib/clore_partner.py b/lib/clore_partner.py index f9dcc12..314cc5b 100644 --- a/lib/clore_partner.py +++ b/lib/clore_partner.py @@ -117,7 +117,7 @@ async def configure(partner_config): partner_container_config = { "image": partner_config["partner_image"], "name": config.clore_partner_container_name, - "hostname": f"{partner_config['partner_id'][:4]}-m{partner_config['machine_id']}", + "hostname": f"{partner_config['partner_id'][:16]}-m{partner_config['machine_id']}", "env": { "AUTH": partner_config['partner_id'], "ip_addr": partner_config['openvpn_host'], diff --git a/lib/clore_partner_socket.py b/lib/clore_partner_socket.py index aab16c4..66f5dce 100644 --- a/lib/clore_partner_socket.py +++ b/lib/clore_partner_socket.py @@ -1,5 +1,6 @@ from lib import config as config_module from lib import logging as logging_lib +from lib import background_job from lib import utils import asyncio import json @@ -61,6 +62,12 @@ async def handle_client(reader, writer): writer.write(json.dumps({ "can_deploy": can_deploy }).encode()) + elif "stop_background_job" in parsed_data and "time" in parsed_data: + try: + if isinstance(parsed_data["time"], int): + background_job.temporarly_disable(parsed_data["time"]) + except Exception as e: + pass else: writer.write('?'.encode()) await writer.drain() diff --git a/lib/docker_deploy.py b/lib/docker_deploy.py index a6f62a0..8116085 100644 --- a/lib/docker_deploy.py +++ b/lib/docker_deploy.py @@ -1,6 +1,7 @@ from lib import config as config_module from lib import logging as logging_lib from lib import docker_cli_wrapper +from lib import background_job from lib import docker_interface from lib import clore_partner from lib import get_specs @@ -148,7 +149,7 @@ def deploy(validated_containers, allowed_running_containers=[], can_run_partner_ container_options["shm_size"] = f"{SHM_SIZE}m" - if not validated_container["name"] in created_container_names and image_ready: + if not validated_container["name"] in created_container_names and image_ready and not (not background_job.is_enabled() and background_job.is_background_job_container_name(validated_container["name"])): if config.creation_engine == "wrapper": docker_cli_wrapper.create_container(container_options, ip=(validated_container["ip"] if "ip" in validated_container else None), shm_size=SHM_SIZE, docker_gpus=docker_gpus) else: @@ -171,7 +172,10 @@ def deploy(validated_containers, allowed_running_containers=[], can_run_partner_ all_running_container_names.append(container.name) else: all_stopped_container_names.append(container.name) - if container.name in needed_running_names and container.status != 'running': + if background_job.is_background_job_container_name(container.name) and not background_job.is_enabled(): + if container.status == "running": + container.stop() + elif container.name in needed_running_names and container.status != 'running': try: attached_networks = container.attrs['NetworkSettings']['Networks'] if "bridge" in attached_networks.keys() or len(attached_networks.keys())==0: # Ip was not attached, remove container