diff --git a/ansible/playbooks/check_images.yml b/ansible/playbooks/check_images.yml new file mode 100644 index 0000000..34f8a31 --- /dev/null +++ b/ansible/playbooks/check_images.yml @@ -0,0 +1,5 @@ +- name: check images on slave nodes + hosts: fields + gather_facts: False + roles: + - { role: ../roles/images } diff --git a/ansible/playbooks/check_internet_slaves.yml b/ansible/playbooks/check_internet_slaves.yml new file mode 100644 index 0000000..5741c7b --- /dev/null +++ b/ansible/playbooks/check_internet_slaves.yml @@ -0,0 +1,5 @@ +- name: check internet on slave nodes + hosts: fields + gather_facts: False + roles: + - { role: ../roles/internet } diff --git a/ansible/playbooks/check_master.yml b/ansible/playbooks/check_master.yml new file mode 100644 index 0000000..31f8ffa --- /dev/null +++ b/ansible/playbooks/check_master.yml @@ -0,0 +1,4 @@ +- name: ping master node + hosts: master + roles: + - { role: ../roles/ping } \ No newline at end of file diff --git a/ansible/playbooks/check_memory.yml b/ansible/playbooks/check_memory.yml new file mode 100644 index 0000000..8c1eed1 --- /dev/null +++ b/ansible/playbooks/check_memory.yml @@ -0,0 +1,5 @@ +- name: check ROM memory on slave nodes + hosts: fields + gather_facts: False + roles: + - { role: ../roles/memory } diff --git a/ansible/playbooks/check_module.yml b/ansible/playbooks/check_module.yml new file mode 100644 index 0000000..935640e --- /dev/null +++ b/ansible/playbooks/check_module.yml @@ -0,0 +1,5 @@ +- name: check modules running on slave nodes + hosts: fields + gather_facts: False + roles: + - { role: ../roles/check } diff --git a/ansible/playbooks/config_cluster.yml b/ansible/playbooks/config_cluster.yml new file mode 100644 index 0000000..89af95c --- /dev/null +++ b/ansible/playbooks/config_cluster.yml @@ -0,0 +1,5 @@ +- name: check docker engine + hosts: fields + gather_facts: False + roles: + - { role: ../roles/docker } diff --git a/ansible/playbooks/delete_images.yml b/ansible/playbooks/delete_images.yml new file mode 100644 index 0000000..cb9e857 --- /dev/null +++ b/ansible/playbooks/delete_images.yml @@ -0,0 +1,5 @@ +- name: delete images on slave nodes + hosts: fields + gather_facts: False + roles: + - { role: ../roles/delete } diff --git a/ansible/playbooks/login_slaves.yml b/ansible/playbooks/login_slaves.yml new file mode 100644 index 0000000..37c4091 --- /dev/null +++ b/ansible/playbooks/login_slaves.yml @@ -0,0 +1,8 @@ +- name: login on slave nodes + hosts: fields + gather_facts: False + vars: + token: "{{ token }}" + user: "{{ user }}" + roles: + - { role: ../roles/login } diff --git a/ansible/playbooks/more_check.yml b/ansible/playbooks/more_check.yml new file mode 100644 index 0000000..1b4d8dd --- /dev/null +++ b/ansible/playbooks/more_check.yml @@ -0,0 +1,5 @@ +- name: check utilization of slave nodes + hosts: fields + gather_facts: False + roles: + - { role: ../roles/more_check } diff --git a/ansible/playbooks/ping_slaves.yml b/ansible/playbooks/ping_slaves.yml new file mode 100644 index 0000000..de2f12e --- /dev/null +++ b/ansible/playbooks/ping_slaves.yml @@ -0,0 +1,5 @@ +- name: ping slave nodes + hosts: fields + gather_facts: False + roles: + - { role: ../roles/ping } diff --git a/ansible/playbooks/pull_image.yml b/ansible/playbooks/pull_image.yml new file mode 100644 index 0000000..0422fb9 --- /dev/null +++ b/ansible/playbooks/pull_image.yml @@ -0,0 +1,7 @@ +- name: pull image on slave nodes + hosts: fields + gather_facts: False + vars: + image: "{{ image }}" + roles: + - { role: ../roles/image } diff --git a/ansible/playbooks/run_module.yml b/ansible/playbooks/run_module.yml new file mode 100644 index 0000000..a498379 --- /dev/null +++ b/ansible/playbooks/run_module.yml @@ -0,0 +1,11 @@ +- name: run module on slave node + hosts: fields + gather_facts: False + vars: + image: "{{ image }}" + port: "{{ port }}" + size: "{{ size }}" + vars_files: + - ../env/minio_params.yml + roles: + - { role: ../roles/run } diff --git a/ansible/playbooks/stop_module.yml b/ansible/playbooks/stop_module.yml new file mode 100644 index 0000000..00b69d6 --- /dev/null +++ b/ansible/playbooks/stop_module.yml @@ -0,0 +1,7 @@ +- name: stop module on slave node + hosts: fields + gather_facts: False + vars: + module_port: "{{ module_port }}" + roles: + - { role: ../roles/stop } diff --git a/ansible/roles/check/tasks/main.yml b/ansible/roles/check/tasks/main.yml new file mode 100644 index 0000000..1ac139e --- /dev/null +++ b/ansible/roles/check/tasks/main.yml @@ -0,0 +1,7 @@ +- name: Check running modules + ansible.builtin.shell: docker ps -q | wc -l + register: running_info + +- name: Display running info + ansible.builtin.debug: + var: running_info.stdout_lines diff --git a/ansible/roles/delete/tasks/main.yml b/ansible/roles/delete/tasks/main.yml new file mode 100644 index 0000000..fa8c239 --- /dev/null +++ b/ansible/roles/delete/tasks/main.yml @@ -0,0 +1,7 @@ +- name: Delete images + ansible.builtin.shell: docker rmi -f $(docker images -aq) + register: delete_info + +- name: Display delete info + ansible.builtin.debug: + var: delete_info.stdout_lines diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml new file mode 100644 index 0000000..a211366 --- /dev/null +++ b/ansible/roles/docker/tasks/main.yml @@ -0,0 +1,48 @@ +- name: Check docker info + ansible.builtin.shell: docker info + register: docker_info + +- name: Display docker info + ansible.builtin.debug: + var: docker_info.stdout_lines +#- name: Open TCP port 2376 +# ansible.builtin.shell: sudo iptables -I INPUT -p tcp --dport 2376 -j ACCEPT +# register: docker_port + +#- name: Display info for TCP port 2376 +# ansible.builtin.debug: +# var: docker_port.stdout_lines + +#- name: Open TCP port 7946 +# ansible.builtin.shell: sudo iptables -I INPUT -p tcp --dport 7946 -j ACCEPT +# register: tcp_node_port + +#- name: Display info for TCP port 7946 +# ansible.builtin.debug: +# var: tcp_node_port.stdout_lines + +#- name: Open UDP port 7946 +# ansible.builtin.shell: sudo iptables -I INPUT -p udp --dport 7946 -j ACCEPT +# register: udp_node_port + +#- name: Display info for UDP port 7946 +# ansible.builtin.debug: +# var: udp_node_port.stdout_lines + +#- name: Open UDP port 4789 +# ansible.builtin.shell: sudo iptables -I INPUT -p udp --dport 4789 -j ACCEPT +# register: data_port + +#- name: Display info for UDP port 4789 +# ansible.builtin.debug: +# var: data_port.stdout_lines +#- name: Leave old swarm +# ansible.builtin.shell: docker swarm leave +# register: delete_node +#- name: Add node to cluster +# ansible.builtin.shell: docker swarm join --token SWMTKN-1-1ccvjacvym3mw3ikft5p847icyl75oicbd4xeck65s3is5a46r-4rngzt3pnidf57yyvyjq350u2 172.16.150.200:2377 +# register: added_info + +#- name: Display info about added node +# ansible.builtin.debug: +# var: added_info.stdout_lines diff --git a/ansible/roles/image/tasks/main.yml b/ansible/roles/image/tasks/main.yml new file mode 100644 index 0000000..73f6e5a --- /dev/null +++ b/ansible/roles/image/tasks/main.yml @@ -0,0 +1,7 @@ +- name: Pull image + ansible.builtin.shell: docker pull {{ image }} + register: image_info + +- name: Display image info + ansible.builtin.debug: + var: image_info.stdout_lines diff --git a/ansible/roles/images/tasks/main.yml b/ansible/roles/images/tasks/main.yml new file mode 100644 index 0000000..fee7cb0 --- /dev/null +++ b/ansible/roles/images/tasks/main.yml @@ -0,0 +1,7 @@ +- name: Check images + ansible.builtin.shell: docker images + register: images_info + +- name: Display images info + ansible.builtin.debug: + var: images_info.stdout_lines diff --git a/ansible/roles/internet/tasks/main.yml b/ansible/roles/internet/tasks/main.yml new file mode 100644 index 0000000..868ba96 --- /dev/null +++ b/ansible/roles/internet/tasks/main.yml @@ -0,0 +1,7 @@ +- name: Check internet access + ansible.builtin.shell: ping -c 5 google.com + register: internet_info + +- name: Display internet info + ansible.builtin.debug: + var: internet_info.stdout_lines diff --git a/ansible/roles/login/tasks/main.yml b/ansible/roles/login/tasks/main.yml new file mode 100644 index 0000000..91aee8f --- /dev/null +++ b/ansible/roles/login/tasks/main.yml @@ -0,0 +1,7 @@ +- name: Registry login + ansible.builtin.shell: docker login -u {{ user }} -p {{ token }} + register: login_info + +- name: Display login info + ansible.builtin.debug: + var: login_info.stdout_lines diff --git a/ansible/roles/memory/tasks/main.yml b/ansible/roles/memory/tasks/main.yml new file mode 100644 index 0000000..d47db84 --- /dev/null +++ b/ansible/roles/memory/tasks/main.yml @@ -0,0 +1,7 @@ +- name: Check ROM memory + ansible.builtin.shell: df -h + register: memory_info + +- name: Display memory info + ansible.builtin.debug: + var: memory_info.stdout_lines diff --git a/ansible/roles/more_check/tasks/main.yml b/ansible/roles/more_check/tasks/main.yml new file mode 100644 index 0000000..7aab7be --- /dev/null +++ b/ansible/roles/more_check/tasks/main.yml @@ -0,0 +1,41 @@ +- name: Check running modules + ansible.builtin.shell: docker ps -q | wc -l + register: running_info + +- name: Check used system memory + ansible.builtin.shell: free -m | awk 'NR==2{printf "%d", ($2-$3)}' + register: sys_mem_info + +- name: Delete bm-smi log + ansible.builtin.shell: rm -f bm-smi.log + register: del_file + +- name: Save bm-smi log + ansible.builtin.shell: /opt/sophon/libsophon-0.4.8/bin/bm-smi -noloop --file=./bm-smi.log + register: save_file + +- name: Check used device memory + ansible.builtin.shell: cat bm-smi.log | grep -o '[0-9]\+MB/ ' | grep -o -E '[0-9]+' + register: dev_mem_info + +- name: Check used TPU + ansible.builtin.shell: cat bm-smi.log | grep -o '[0-9]\+%' | grep -o -E -m 1 '[0-9]+' + register: tpu_info + +- name: Check used CPU + ansible.builtin.shell: sar 1 1 | grep "Average" | awk '{print 100-$8}' + register: cpu_info + +- name: Check containers ID + ansible.builtin.shell: docker ps -a -q + register: containers_id + +- name: Display state info + ansible.builtin.debug: + msg: + running: "{{ running_info.stdout_lines }}" + sys_mem: "{{ sys_mem_info.stdout_lines }}" + dev_mem: "{{ dev_mem_info.stdout_lines }}" + tpu: "{{ tpu_info.stdout }}" + cpu: "{{ cpu_info.stdout }}" + containers_ids: "{{ containers_id.stdout_lines }}" diff --git a/ansible/roles/run/tasks/main.yml b/ansible/roles/run/tasks/main.yml new file mode 100644 index 0000000..018fd84 --- /dev/null +++ b/ansible/roles/run/tasks/main.yml @@ -0,0 +1,30 @@ +- name: Create model weights directory if it does not exist + ansible.builtin.file: + path: /data/weights/synapse-faces-weights + state: directory + mode: '0775' + +- name: Run module + ansible.builtin.shell: docker run \ + -d \ + --rm \ + --privileged=true \ + -v /opt/sophon:/opt/sophon \ + -v /etc/profile.d:/etc/profile.d \ + -v /etc/ld.so.conf.d:/etc/ld.so.conf.d \ + -v /data/weights/synapse-faces-weights:/server/data/weights/int8 \ + -e DETECTOR_SIZE={{ size }} \ + -e TZ="Europe/Moscow" \ + -e MINIO_ENDPOINT="{{ MINIO_ENDPOINT }}" \ + -e MINIO_ACCESS_KEY="{{ MINIO_ACCESS_KEY }}" \ + -e MINIO_SECRET_KEY="{{ MINIO_SECRET_KEY }}" \ + -e MINIO_PUB_CERT="{{ MINIO_PUB_CERT }}" \ + -p {{ port }}:9000 \ + --workdir /server \ + --name faces_service_{{ port }} {{ image }} /bin/bash docker/tpu/entry.sh + + register: run_info + +- name: Display run info + ansible.builtin.debug: + var: run_info.stdout_lines diff --git a/ansible/roles/stop/tasks/main.yml b/ansible/roles/stop/tasks/main.yml new file mode 100644 index 0000000..83e175f --- /dev/null +++ b/ansible/roles/stop/tasks/main.yml @@ -0,0 +1,7 @@ +- name: Stop module + ansible.builtin.shell: docker stop faces_service_{{ module_port }} + register: stop_info + +- name: Display stop info + ansible.builtin.debug: + var: stop_info.stdout_lines