[Python] I/O Bound(2) - Multiproecessing

통신관련 멀티프로세스 처리(동기방식)

 

 

import multiprocessing
import requests
import time

# 각 프로세스 메모리 영역에 생성되는 객체(독립적)
# 함수 실행 할 때 마다 객체 생성은 좋지 않음. -> 각 프로세스마다 할당
session = None

def set_global_session():
    global session
    if not session:
        session = requests.Session()
        print('>>> not session()')


def request_site(url):
    # 세션 확인
    print(session)
    # print(session.headers)

    with session.get(url) as response:
        name = multiprocessing.current_process().name
        print(f"[{name} -> Read Contents : {len(response.content)}, Status Code : {response.status_code}] from {url}")
        

def request_all_site(urls):
    # 멀티프로세싱 실행
    # 반드시 processes 개수 조절 후 session 객체 및 실행 시간 확인
    # 생략시 자동으로 설정
    with multiprocessing.Pool(initializer=set_global_session, processes=3) as pool:
        # pool 병렬화(함수 실행)
        pool.map(request_site, urls)

def main():

    urls = [
            "https://www.jython.org",
            "https://www.naver.com",
            "https://realpython.com/"
    ] * 3
    
    start_time = time.time()
    request_all_site(urls)
    duration = time.time() - start_time
    
    print(f"Downloaded {len(urls)} sites in {duration} seconds")

if __name__ == "__main__":
    main()
    
--------------------------------------------[result]    
>>> not session()
<requests.sessions.Session object at 0x000001A2BF72CD90>
[SpawnPoolWorker-3 -> Read Contents : 40221, Status Code : 200] from https://realpython.com/
<requests.sessions.Session object at 0x000001A2BF72CD90>
[SpawnPoolWorker-3 -> Read Contents : 272, Status Code : 200] from http://olympus.realpython.org/dice
>>> not session()
<requests.sessions.Session object at 0x000001D0A87ACD90>
[SpawnPoolWorker-1 -> Read Contents : 10394, Status Code : 200] from https://www.jython.org
<requests.sessions.Session object at 0x000001D0A87ACD90>
[SpawnPoolWorker-1 -> Read Contents : 10394, Status Code : 200] from https://www.jython.org
<requests.sessions.Session object at 0x000001D0A87ACD90>
[SpawnPoolWorker-1 -> Read Contents : 40219, Status Code : 200] from https://realpython.com/
>>> not session()
<requests.sessions.Session object at 0x000001E760ADCD90>
[SpawnPoolWorker-2 -> Read Contents : 272, Status Code : 200] from http://olympus.realpython.org/dice
<requests.sessions.Session object at 0x000001E760ADCD90>
[SpawnPoolWorker-2 -> Read Contents : 272, Status Code : 200] from http://olympus.realpython.org/dice
<requests.sessions.Session object at 0x000001E760ADCD90>
[SpawnPoolWorker-2 -> Read Contents : 10394, Status Code : 200] from https://www.jython.org
<requests.sessions.Session object at 0x000001E760ADCD90>
[SpawnPoolWorker-2 -> Read Contents : 40221, Status Code : 200] from https://realpython.com/

Downloaded 9 sites in 1.5694267749786377 seconds