파이썬 병렬 처리(멀티프로세싱)이 안 되네요...

조회수 41회
import pandas as pd
import numpy as np
import openpyxl
import pymysql
from dateutil.relativedelta import relativedelta
from datetime import datetime
import time
from multiprocessing import Process, Queue
import os

conn = pymysql.connect(host = "xxxx", port=3306, user = "root", password = "xxxx", db = "12MFWModel3")
try:
    with conn.cursor() as cursor:
        sql = '''
            CREATE TABLE time (
                code text, name text, OPScore int(11), NPScore int(11), OPMScore int(11), ROEScore int(11), SalesScore int(11), TotalScore int(11),
                year int(11), month int(11),day int(11)
            ) ENGINE=InnoDB DEFAULT CHARSET=utf8
'''
        cursor.execute(sql)
    conn.commit()

finally:
    conn.close()

def work(month):
    if month != 12:
        index1 = (datetime(2019,month,1) - datetime(2019,1,1)).days
        index2 = (datetime(2019,month+1,1) - datetime(2019,1,1)).days
    else:
        index1 = (datetime(2019,12,1) - datetime(2019,1,1)).days
        index2 = (datetime(2020,1,1) - datetime(2019,1,1)).days

    for i in range(index1, index2):
        Mdate = date2[i] - relativedelta(months=1)
        year2 = date2[i].year
        month2 = date2[i].month
        day2 = date2[i].day
        year1 = Mdate.year
        month1 = Mdate.month
        day1 = Mdate.day

        OPScore = []
        NPScore = []
        OPMScore = []
        ROEScore = []
        SalesScore = []
        TotalScore = []

        conn = pymysql.connect(host = "xxxx", port=3306, user = "root", password = "xxxx", db = "12MFWModel3")

        for k in range (0,len(Code)):
            OP2 = np.array(FW12MOP2019.iloc[13:,1:])[i][k]
            if year1 == year2 - 1:
                j = (Mdate - date1[0]).days
                OP1 = np.array(FW12MOP2018.iloc[13:,1:])[j][k]
            if year1 == year2:
                j = (Mdate - date2[0]).days
                OP1 = np.array(FW12MOP2019.iloc[13:,1:])[j][k]
            if not ((type(OP1) is float)|(type(OP1) is int)) & ((type(OP2) is float)|(type(OP2) is int)):
                CHOP = None
            else:
                if OP1<0:
                    if OP2<=0:
                        CHOP = (OP1-OP2)/OP1
                    else:
                        CHOP = 999
                else: 
                    if OP1==0:
                        CHOP = "Error"
                    else:
                        CHOP = (OP2-OP1)/OP1

            if (type(CHOP) is float)|(type(CHOP) is int):
                if CHOP>=0.05:
                    OPScore.append(1)
                else:
                    OPScore.append(0)
            else:
                OPScore.append(0)

            NP2 = np.array(FW12MNP2019.iloc[13:,1:])[i][k]
            if year1 == year2 - 1:
                j = (Mdate - date1[0]).days
                NP1 = np.array(FW12MNP2018.iloc[13:,1:])[j][k]
            if year1 == year2:
                j = (Mdate - date2[0]).days
                NP1 = np.array(FW12MNP2019.iloc[13:,1:])[j][k]
            if not ((type(NP1) is float)|(type(NP1) is int)) & ((type(NP2) is float)|(type(NP2) is int)):
                CHNP = None
            else:
                if NP1<0:
                    if NP2<=0:
                        CHNP = (NP1-NP2)/NP1
                    else:
                        CHNP = 999
                else:
                    if NP1==0:
                        CHNP = "Error"
                    else:
                        CHNP = (NP2-NP1)/NP1

            if (type(CHNP) is float)|(type(CHNP) is int):
                if CHNP>=0.05:
                    NPScore.append(1)
                else:
                    NPScore.append(0)
            else:
                NPScore.append(0)

            Sales2 = np.array(FW12MSales2019.iloc[13:,1:])[i][k]
            if year1 == year2 - 1:
                j = (Mdate - date1[0]).days
                Sales1 = np.array(FW12MSales2018.iloc[13:,1:])[j][k]
            if year1 == year2:
                j = (Mdate - date2[0]).days
                Sales1 = np.array(FW12MSales2019.iloc[13:,1:])[j][k]
            if not ((type(Sales1) is float)|(type(Sales1) is int)) & ((type(Sales2) is float)|(type(Sales2) is int)):
                CHSales = None
            else:
                if Sales1<0:
                    if Sales2<=0:
                        CHSales = (Sales1-Sales2)/Sales1
                    else:
                        CHSales = 999
                else:
                    if Sales1==0:
                        CHSales = "Error"
                    else:
                        CHSales = (Sales2-Sales1)/Sales1

            if (type(CHSales) is float)|(type(CHSales) is int):
                if CHSales>=0.05:
                    SalesScore.append(1)
                else:
                    SalesScore.append(0)
            else:
                SalesScore.append(0)

            if not ((type(np.array(FW12MSales2019.iloc[13:,1:])[i][k]) is float)|(type(np.array(FW12MSales2019.iloc[13:,1:])[i][k]) is int)) & ((type(np.array(FW12MOP2019.iloc[13:,1:])[i][k]) is float)|(type(np.array(FW12MOP2019.iloc[13:,1:])[i][k]) is int)):
                OPM2 = None
            else:
                OPM2 = np.array(FW12MOP2019.iloc[13:,1:])[i][k]/np.array(FW12MSales2019.iloc[13:,1:])[i][k]

            if year1 == year2 - 1:
                j = (Mdate - date1[0]).days
                if not ((type(np.array(FW12MSales2018.iloc[13:,1:])[j][k]) is float)|(type(np.array(FW12MSales2018.iloc[13:,1:])[j][k]) is int)) & ((type(np.array(FW12MOP2018.iloc[13:,1:])[j][k]) is float)|(type(np.array(FW12MOP2018.iloc[13:,1:])[j][k]) is int)):
                    OPM1 = None
                else:
                    OPM1 = np.array(FW12MOP2018.iloc[13:,1:])[j][k]/np.array(FW12MSales2018.iloc[13:,1:])[j][k]
            if year1 == year2:
                j = (Mdate - date2[0]).days
                if not ((type(np.array(FW12MSales2019.iloc[13:,1:])[j][k]) is float)|(type(np.array(FW12MSales2019.iloc[13:,1:])[j][k]) is int)) & ((type(np.array(FW12MOP2019.iloc[13:,1:])[j][k]) is float)|(type(np.array(FW12MOP2019.iloc[13:,1:])[j][k]) is int)):
                    OPM1 = None
                else:
                    OPM1 = np.array(FW12MOP2019.iloc[13:,1:])[j][k]/np.array(FW12MSales2019.iloc[13:,1:])[j][k]

            if not ((type(OPM1) is float)|(type(OPM1) is int)) & ((type(OPM2) is float)|(type(OPM2) is int)):
                CHOPM = None
            else:
                if OPM1<0:
                    if OPM2<=0:
                        CHOPM = (OPM1-OPM2)/OPM1
                    else:
                        CHOPM = 999
                else:
                    if OPM1==0:
                        CHOPM = "Error"
                    else:
                        CHOPM = (OPM2-OPM1)/OPM1

            if (type(CHOPM) is float)|(type(CHOPM) is int):
                if CHOPM>=0.05:
                    OPMScore.append(1)
                else:
                    OPMScore.append(0)
            else:
                OPMScore.append(0)


            if not (type(np.array(Equity20192018Con.iloc[13:,1:])[i][k]) is float)|(type(np.array(Equity20192018Con.iloc[13:,1:])[i][k]) is int):
                Equity2018 = list(Equity20192018.iloc[5:,2])[i]
            else:
                Equity2018 = np.array(Equity20192018Con.iloc[13:,1:])[i][k]

            Equity2019 = np.array(Equity20192019.iloc[13:,1:])[i][k]
            Equity2020 = np.array(Equity20192020.iloc[13:,1:])[i][k]

            if not ((type(Equity2018) is float)|(type(Equity2018) is int))&((type(Equity2019) is float)|(type(Equity2019) is int))&((type(Equity2020) is float)|(type(Equity2020) is int)):
                Equity2 = None
            else:
                Equity2 = (i/365*Equity2018+Equity2019+(365-i)/365*Equity2020)/2            

            if year1 == year2 - 1:
                j = (Mdate - date1[0]).days
                Equity2017 = list(Equity20182017.iloc[5:,2])[j]
                Equity2018 = np.array(Equity20182018.iloc[13:,1:])[j][k]
                Equity2019 = np.array(Equity20182019.iloc[13:,1:])[j][k]
                if not ((type(Equity2017) is float)|(type(Equity2017) is int))&((type(Equity2018) is float)|(type(Equity2018) is int))&((type(Equity2019) is float)|(type(Equity2019) is int)):
                    Equity1 = None
                else:
                    Equity1 = (j/365*Equity2017+Equity2018+(365-j)/365*Equity2019)/2      

            if year1 == year2:
                j = (Mdate - date2[0]).days
                if not (type(np.array(Equity20192018Con.iloc[13:,1:])[j][k]) is float)|(type(np.array(Equity20192018Con.iloc[13:,1:])[j][k]) is int):
                    Equity2018 = list(Equity20192018.iloc[5:,2])[j]
                else:
                    Equity2018 = np.array(Equity20192018Con.iloc[13:,1:])[j][k]

                Equity2019 = np.array(Equity20192019.iloc[13:,1:])[j][k]
                Equity2020 = np.array(Equity20192020.iloc[13:,1:])[j][k]

                if not ((type(Equity2018) is float)|(type(Equity2018) is int))&((type(Equity2019) is float)|(type(Equity2019) is int))&((type(Equity2020) is float)|(type(Equity2020) is int)):
                    Equity1 = None
                else:
                    Equity1 = (j/365*Equity2018+Equity2019+(365-j)/365*Equity2020)/2 

            if not ((type(NP2) is float)|(type(NP2) is int)) & ((type(Equity2) is float)|(type(Equity2) is int)):
                ROE2 = None
            else:
                ROE2 = NP2/Equity2
            if not ((type(NP1) is float)|(type(NP1) is int)) & ((type(Equity1) is float)|(type(Equity1) is int)):
                ROE1 = None
            else:
                ROE1 = NP1/Equity1

            if not ((type(ROE1) is float)|(type(ROE1) is int)) & ((type(ROE2) is float)|(type(ROE2) is int)):
                CHROE = None
            else:
                if ROE1<0:
                    if ROE2<=0:
                        CHROE = (ROE1-ROE2)/ROE1
                    else:
                        CHROE = 999
                else:
                    if ROE1==0:
                        CHROE = "Error"
                    else:
                        CHROE = (ROE2-ROE1)/ROE1

            if (type(CHROE) is float)|(type(CHROE) is int):
                if CHROE>=0.05:
                    ROEScore.append(1)
                else:
                    ROEScore.append(0)
            else:
                ROEScore.append(0)

            TotalScore.append(OPScore[k]+NPScore[k]+OPMScore[k]+ROEScore[k]+SalesScore[k])

            sql = """INSERT INTO time(code, name, OPScore, NPScore, OPMScore, ROEScore, SalesScore, TotalScore, year, month, day) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"""
            values = (Code[k], Name[k], OPScore[k], NPScore[k], OPMScore[k], ROEScore[k], SalesScore[k], TotalScore[k], year2, month2, day2)
            curs = conn.cursor()
            curs.execute(sql, values)
        conn.commit()
        conn.close()

if __name__ == '__main__':

    procs = []
    for s in (1,13):
        proc = Process(target = work, args=(s,))
        procs.append(proc)
        proc.start()

    for proc in procs:
        proc.join()

구글링 해서 멀티프로세싱 방법을 구현하였는데 실행시키면 바로 종료됩니다. 마지막 메인 코드를 보시면 12개의 과정(1월~12월)을 병렬로 돌려서 mysql table에 업로드 시키는 것이 제가 원하는 목표인데 데이터가 하나도 올라가지 않고 종료됩니다.

이유를 알려주시면 정말 감사드리겠습니다. 참고로 에러는 안 나서 뭐가 잘못된 건지도 모르겠습니다

  • db처리 다 빼시고 multiprocess 만 되는지 확인하세요. db처리 때문에 오류가 발생하는지부터 확인하는 편이 낫겠죠. 정영훈 2019.12.2 10:55

답변을 하려면 로그인이 필요합니다.

Hashcode는 개발자들을 위한 무료 QnA 사이트입니다. 계정을 생성하셔야만 답변을 작성하실 수 있습니다.

(ಠ_ಠ)
(ಠ‿ಠ)

ᕕ( ᐛ )ᕗ
로그인이 필요합니다

Hashcode는 개발자들을 위한 무료 QnA사이트 입니다. 계정을 생성하셔야만 글을 작성하실 수 있습니다.