Adding CPU Affinity to Jobs

README.RU.md

@@ -174,6 +174,10 @@ List of Metrics Gathered
 * `concurrency_start_early_phase` - Фаза, в которой вы хотите начать засеивание заранее. Рекомендуется использовать 4.
 * `concurrency_start_early_phase_delay` - Максимальное количество минут ожидания до запуска нового участка при обнаружении ранней фазы запуска.
 * `temporary2_destination_sync` - Представлять каталог назначения как каталог второй временный каталог. Эти два каталога будут синхронизированы, так что они всегда будут представлены как одно и то же значение.
+* `exclude_final_directory` - Whether to skip adding `destination_directory` to harvester for farming. This is a Chia feature.
+* `enable_cpu_affinity` - Enable or disable cpu affinity for plot processes. Systems that plot and harvest may see improved harvester or node performance when excluding one or two threads for plotting process.
+* `cpu_affinity` - List of cpu (or threads) to allocate for plot processes. The default example assumes you have a hyper-threaded 4 core CPU (8 logical cores). This config will restrict plot processes to use logical cores 0-5, leaving logical cores 6 and 7 for other processes (6 restricted, 2 free).
+
 
 ### Перевод на Русский
 Оригинальный текст на Английском языке Вы можете найти по адресу [https://github.com/swar/Swar-Chia-Plot-Manager](https://github.com/swar/Swar-Chia-Plot-Manager)

README.md

@@ -175,4 +175,6 @@ Check for more details on the Chia CLI here: https://github.com/Chia-Network/chi
 * `concurrency_start_early_phase` - The phase in which you want to start a plot early. It is recommended to use 4 for this field.
 * `concurrency_start_early_phase_delay` - The maximum number of minutes to wait before a new plot gets kicked off when the start early phase has been detected.
 * `temporary2_destination_sync` - This field will always submit the destination directory as the temporary2 directory. These two directories will be in sync so that they will always be submitted as the same value.
-* `exclude_final_directory` - Whether to skip adding `destination_directory` to harvester for farming
+* `exclude_final_directory` - Whether to skip adding `destination_directory` to harvester for farming. This is a Chia feature.
+* `enable_cpu_affinity` - Enable or disable cpu affinity for plot processes. Systems that plot and harvest may see improved harvester or node performance when excluding one or two threads for plotting process.
+* `cpu_affinity` - List of cpu (or threads) to allocate for plot processes. The default example assumes you have a hyper-threaded 4 core CPU (8 logical cores). This config will restrict plot processes to use logical cores 0-5, leaving logical cores 6 and 7 for other processes (6 restricted, 2 free).

config.yaml.default

@@ -57,7 +57,7 @@ notifications:
   notify_pushover: false
   pushover_user_key: xx
   pushover_api_key: xx
-  
+
   # TWILIO
   notify_twilio: false
   twilio_account_sid: xxxxx
@@ -139,6 +139,11 @@ jobs:
   #                              These two directories will be in sync so that they will always be submitted as the
   #                              same value.
   # exclude_final_directory: Whether to skip adding `destination_directory` to harvester for farming
+  # enable_cpu_affinity: Enable or disable cpu affinity for plot processes. Systems that plot and harvest may see
+  #                      improved harvester or node performance when excluding one or two threads for plotting process.
+  #        cpu_affinity: List of cpu (or threads) to allocate for plot processes. The default example assumes you have
+  #                      a hyper-threaded 4 core CPU (8 logical cores). This config will restrict plot processes to use
+  #                      logical cores 0-5, leaving logical cores 6 and 7 for other processes (6 restricted, 2 free).
   - name: micron
     max_plots: 999
     farmer_public_key:
@@ -159,6 +164,8 @@ jobs:
     concurrency_start_early_phase_delay: 0
     temporary2_destination_sync: false
     exclude_final_directory: false
+    enable_cpu_affinity: false
+    cpu_affinity: [ 0, 1, 2, 3, 4, 5 ]
 
   - name: inland
     max_plots: 999
@@ -184,3 +191,5 @@ jobs:
     concurrency_start_early_phase_delay: 0
     temporary2_destination_sync: false
     exclude_final_directory: false
+    enable_cpu_affinity: false
+    cpu_affinity: [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 ]

plotmanager/library/utilities/commands.py

@@ -29,7 +29,7 @@ def start_manager():
     manager_log_file = open(manager_log_file_path, 'a')
     python_file_path = sys.executable
 
-    chia_location, log_directory, jobs, manager_check_interval, max_concurrent, progress_settings, \
+    chia_location, log_directory, config_jobs, manager_check_interval, max_concurrent, progress_settings, \
         notification_settings, debug_level, view_settings, instrumentation_settings = get_config_info()
 
     test_configuration(chia_location=chia_location, notification_settings=notification_settings,
@@ -132,6 +132,6 @@ def view():
 
 
 def analyze_logs():
-    chia_location, log_directory, jobs, manager_check_interval, max_concurrent, progress_settings, \
-       notification_settings, debug_level, view_settings, instrumentation_settings = get_config_info()
+    chia_location, log_directory, config_jobs, manager_check_interval, max_concurrent, progress_settings, \
+        notification_settings, debug_level, view_settings, instrumentation_settings = get_config_info()
     analyze_log_times(log_directory)

plotmanager/library/utilities/jobs.py

@@ -96,6 +96,11 @@ def load_jobs(config_jobs):
         job.threads = info['threads']
         job.buckets = info['buckets']
         job.memory_buffer = info['memory_buffer']
+
+        job.enable_cpu_affinity = info.get('enable_cpu_affinity', False)
+        if job.enable_cpu_affinity:
+            job.cpu_affinity = info['cpu_affinity']
+
         jobs.append(job)
 
     return jobs
@@ -192,7 +197,13 @@ def monitor_jobs_to_start(jobs, running_work, max_concurrent, next_job_work, chi
         if job.stagger_minutes:
             next_job_work[job.name] = datetime.now() + timedelta(minutes=job.stagger_minutes)
             logging.info(f'Calculating new job stagger time. Next stagger kickoff: {next_job_work[job.name]}')
-        job, work = start_work(job=job, chia_location=chia_location, log_directory=log_directory, drives_free_space=drives_free_space)
+
+        job, work = start_work(
+            job=job,
+            chia_location=chia_location,
+            log_directory=log_directory,
+            drives_free_space=drives_free_space,
+        )
         jobs[i] = deepcopy(job)
         if work is None:
             continue
@@ -257,6 +268,10 @@ def start_work(job, chia_location, log_directory, drives_free_space):
     logging.info(f'Setting priority level: {nice_val}')
     psutil.Process(pid).nice(nice_val)
     logging.info(f'Set priority level')
+    if job.enable_cpu_affinity:
+        logging.info(f'Setting process cpu affinity: {job.cpu_affinity}')
+        psutil.Process(pid).cpu_affinity(job.cpu_affinity)
+        logging.info(f'Set process cpu affinity')
 
     work.pid = pid
     job.total_running += 1

plotmanager/library/utilities/objects.py

@@ -30,6 +30,9 @@ class Job:
     buckets = None
     memory_buffer = None
 
+    enable_cpu_affinity = False
+    cpu_affinity = []
+
 
 class Work:
     work_id = None