From StrategicWiseKing at protonmail.com Fri Nov 15 09:08:58 2024 From: StrategicWiseKing at protonmail.com (Balthazar) Date: Fri, 15 Nov 2024 14:08:58 +0000 Subject: [Tutor] Race condition, Pycharm Handling or something else with Queues and multiprocessing.ThreadPool? Message-ID: <SvkNUGmzOpnA62f2waJtGkVKGKYXz-Ns7ez0x00RMRIbJngoxoHhhDiM50zIckOHbYzRGT11tSihgvZWID_bwrp5bs47RBVi2i0uv2H8dGA=@protonmail.com> Hi, I've been trying to understand ThreadPool and am having a funny time with it. The code runs through partly once and then finishes. However it is meant to repeatedly keep looping. The only way I've seem to get it to work is by running the code in debug on Pycharm and Stepping Over each line.? So why do I think it is a race condition? Because there are still items in the queue before it exits but visually everything seems to be in order, so this becomes a lack of knowledge on my part to understand what is really happening. ---- start here--- import logging from queue import Queue, Empty from multiprocessing.pool import ThreadPool from multiprocessing import Event import time class MultithreadedPoolAndQueue: ? ? def __init__(self, available_modules): ? ? ? ? self.logger = logging.getLogger("Test_Program.MultithreadedPoolAndQueue") ? ? ? ? self.available_modules = available_modules ? ? ? ? self.logger.info("Multithreaded Pool and Queue Management Initialised") ? ? ? ? self.queue = Queue() ? ? ? ? self.pool = ThreadPool(processes=len(self.available_modules)) ? ? ? ? self.all_tasks_completed = Event() ? ? def unpack_results(self,results): ? ? ? ? filtered_results = [] ? ? ? ? for key, val in results.items(): ? ? ? ? ? ? if isinstance(val, list): ? ? ? ? ? ? ? ? for item in val: ? ? ? ? ? ? ? ? ? ? filtered_results.append({key: item}) ? ? ? ? ? ? else: ? ? ? ? ? ? ? ? filtered_results.append({key: val}) ? ? ? ? ? ? ? ? ? ? ? ? return filtered_results ? ? def fill_queue(self, data): ? ? ? ? for d in data: ? ? ? ? ? ? # Each data will be {DATA_TYPE: Data} ? ? ? ? ? ? self.queue.put(d) ? ? def parse_results(self, results): ? ? ? ? """ ? ? ? ? This function will need the filtered results.? ? ? ? It will put the data onto the queue like this: {DATA_TYPE: Data} ? ? ? ? :param?results: Results from function.? ? ? ? :return: ? ? ? ? """? ? ? ? self.logger.info(f"Entering parse_results with: {results}") ? ? ? ? if results is None: ? ? ? ? ? ? pass ? ? ? ? else: ? ? ? ? ? ? unpacked_results = self.unpack_results(results) ? ? ? ? ? ? for result in unpacked_results: ? ? ? ? ? ? ? ? self.logger.info(f"Result added: {result}") ? ? ? ? ? ? ? ? self.queue.put(result) ? ? def error_call(self,e): ? ? ? ? self.logger.error(e) ? ? ? ? pass ? ? def thread_worker(self): ? ? ? ? max_check_count = 10 ? ? ? ? check = 0 ? ? ? ? loop_around = 0 ? ? ? ? while self.queue.empty() is False or loop_around < 2 and check < max_check_count: ? ? ? ? ? ? try: ? ? ? ? ? ? ? ? item = self.queue.get_nowait() ? ? ? ? ? ? ? ? # Assuming item is in the format {data_type: data} ? ? ? ? ? ? ? ? data_type = next(iter(item)) ?# Get the first (and only) key from the dictionary ? ? ? ? ? ? ? ? query = item[data_type] ? ? ? ? ? ? ? ? # Then for each module in available modules, apply an asynchronous system ? ? ? ? ? ? ? ? for module in self.available_modules: ? ? ? ? ? ? ? ? ? ? for module_name, module_class in module.items(): ? ? ? ? ? ? ? ? ? ? ? ? # Check if the module can handle this data type ? ? ? ? ? ? ? ? ? ? ? ? if data_type in module_class.accepted_data: ? ? ? ? ? ? ? ? ? ? ? ? ? ? module_instance = module_class() ? ? ? ? ? ? ? ? ? ? ? ? ? ? self.pool.apply_async(module_instance.event,args=(query, data_type),callback=self.parse_results,error_callback=self.error_call) ? ? ? ? ? ? ? ? loop_around = 0 ? ? ? ? ? ? ? ? check = 0 ?# Reset check counter on successful item processing ? ? ? ? ? ? except Empty: ? ? ? ? ? ? ? ? # If the queue is empty, wait for a short time before checking again ? ? ? ? ? ? ? ? time.sleep(0.1) ? ? ? ? ? ? ? ? check += 1 ? ? ? ? ? ? ? ? loop_around += 1 ? ? ? ? ? ? except Exception as e: ? ? ? ? ? ? ? ? self.logger.exception(e) ? ? ? ? ? ? ? ? loop_around += 1 ? ? ? ? # Wait for all async tasks to complete ? ? ? ? self.pool.close() ? ? ? ? self.pool.join() ? ? ? ? self.all_tasks_completed.set() ? ? def wait_for_completion(self): ? ? ? ? self.all_tasks_completed.wait() ? ? ? ? self.logger.info(f"Queue item: {self.queue.get()}") ? ? ? ? self.logger.info("All tasks have been completed.") ---- end here--- Here's how I've been using the code. ---- start here ---- threaded_core = worker_core.MultithreadedPoolAndQueue(self.available_modules) threaded_core.fill_queue(self.targets) threaded_core.thread_worker() threaded_core.wait_for_completion() ---- end here ---- If you require more code then let me know, thank you for any advice in advance. From mk1853387 at gmail.com Wed Nov 27 07:28:06 2024 From: mk1853387 at gmail.com (marc nicole) Date: Wed, 27 Nov 2024 13:28:06 +0100 Subject: [Tutor] How to break while loop based on events raised in a thread (Python 2.7) Message-ID: <CAGJtH9SuFdPOF80BW2x+obM7-CbaqRuHOnraSmxmD+-yLqOOPw@mail.gmail.com> I am using the below class1 code to detect some event through the method check_for_the_event() # Some class1def check_for_the_event(self): thread1 = threading.Timer(2, self.check_for_the_event) thread1.start()# some processingif event is detected: self.event_ok = Trueelse: self.event_ok = False then I pass an instance of that class to the below class to know when the event is on or off and act upon its value accordingly using the following below code: # execution of other part of the program (where self.another_class_instance.event_ok = False and self.some_other_condition is true) current_time = current_time_some_cond = current_max_time = time.time() while not self.another_class_instance.event_ok and self.some_other_condition: while self.some_other_condition and not self.another_class_instance.event_ok:#self.some_other_condition takes up to 10 secs to be checked (minimum 5 secs) if time.time() > current_time + 5: current_time = time.time() # some processing else: while not self.another_class_instance.event_ok: #processing if time.time() > current_max_time + 60 * 2 and not self.another_class_instance.event_ok: #some other processing if time.time() > current_time_some_cond + 10 and not self.cond1 and not self.another_class_instance.event_ok: # some processing that takes 2-3 seconds self.cond1 = True current_time_some_cond = time.time() elif self.cond1 and time.time() > current_time_some_cond + 10 and not self.another_class_instance.event_ok: current_time_some_cond = time.time() #some other processing that takes 2-3 seconds else: pass else: pass The problem is the real time execution of the program (in class2) requires an instant check to be performed and that cannot wait until the current loop instructions finish, and I want is to break out of the outer while loop if the event is on without further checking any other condition. for now I perform multiple checks at each if or while statement, but is there a IO async based method that breaks out of the loop when the event is raised in the thread?