内存使用

Question

所以我有一些代码可以接收一组文件，将它可以拼接在一起，然后绘制它们。我发布了我的大部分代码，试图使它更具可读性，如果需要可以添加更多

for paths,dirs,files in os.walk(start_path):
for d in dirs:
    path = start_path +  changedir + d
    pathpart = d

    os.chdir(path)
    for file in glob.glob("*-0.dat"):
        tempname = file.split("-")
        fileName1 = str(tempname[0] + "-" + tempname[1]+ "-")
        gc.collect()

        Chan2 = []
        Chan1 = []
        temp_1 = []
        temp_2 = []
        temp_3 = []
        Data_Sets1 = []
        Data_Sets2 = []
        Headers = []


        for fileNumber in range(0,45):
            fileName = fileName1 + str(fileNumber) + fileName3
            header, data1, data2 = u.unpackFile(path,fileName)

            if header == None:
                logging.warning("curropted file found at " + fileName)
                Data_Sets1.append(temp_1)
                Data_Sets2.append(temp_2)
                Headers.append(temp_3)
                temp_1 = []
                temp_2 = []
                temp_3 = []
            else:
                logging.info(fileName + " is good!")
                temp_3.append(header)
                for i in range(0,10000):
                    temp_1.append(data1[i])
                    temp_2.append(data2[i])

        Data_Sets1.append(temp_1)
        Data_Sets2.append(temp_2)
        Headers.append(temp_3)
        temp_1 = []
        temp_2 = []
        temp_3 = []

        del temp_1
        del temp_2
        del temp_3

        lengths = []
        for i in range(len(Data_Sets1)):
            lengths.append(len(Data_Sets1[i]))
        index = lengths.index(max(lengths))

        Chan1 = Data_Sets1[index]
        Chan2 = Data_Sets2[index]
        Start_Header = Headers[index]
        if (len(Chan1) == 0 | len(Chan2) == 0):
            continue
        try:
            Date = Start_Header[index][0]
            Time = Start_Header[index][1]
        except IndexError:
            logging.critical("file " + fileName + " is unusuable")
            continue
        """
        Clostest_Power = int(np.log(len(Chan1))/np.log(2))
        Length = 2 ** Clostest_Power
        logging.debug("Length of the file is " + str(Length))
        Chan1 = Chan1[0:Length]
        Chan2 = Chan2[0:Length]
        """
        logging.debug("Length of channels is " + str(len(Chan1)))   

        window = np.hanning(Window_Width)

        t= s.Time_Array(len(Chan1),Sample_Rate)


        window2 = np.hanning(len(Chan1))

        Noise_Frequincies = []
        for i in range(1,125):
            Noise_Frequincies.append(60.0*float(i))
        Noise_Frequincies.append(180.0)

        filter1 = s.Noise_Reduction(Sample_Rate,Noise_Frequincies,Chan1)
        filter2 = s.Noise_Reduction(Sample_Rate,Noise_Frequincies,Chan2)

        logging.info("Starting the plots")


        fig1, (ax1, ax2) = plt.subplots(nrows=2)


        spec1, freqs1, time1 = mlab.specgram(filter1, NFFT=Window_Width, Fs=Sample_Rate, window=window, noverlap=Over_Lap)


        im1 = ax1.imshow(spec1, cmap=cm.get_cmap("rainbow"), norm=colors.LogNorm(), origin='lower',
            extent=[t[0], t[len(t)-1], freqs1.min(), 8000],aspect='auto',vmin=1e-5,vmax=1e5)

        ax1.set_title(str(Date) + "-" + str(Time) + " Channel 1")
        ax1.set_ylabel("Freqency Hz")



        spec2, freqs2, time2 = mlab.specgram(filter2, NFFT=Window_Width, Fs=Sample_Rate, window=window, noverlap=Over_Lap)

        im2 = ax2.imshow(spec2, cmap=cm.get_cmap("rainbow"), norm=colors.LogNorm(), origin='lower',
            extent=[t[0], t[len(t)-1], freqs2.min(), 8000],aspect='auto',vmin=1e-5,vmax=1e5)

        cax1, kw1 = matplotlib.colorbar.make_axes(ax1)
        colorbar(im1,cax=cax1,**kw1)
        cax2, kw2 = matplotlib.colorbar.make_axes(ax2)
        colorbar(im2,cax=cax2,**kw2)

        ax2.set_title(str(Date) + "-" + str(Time) + " Channel 2")
        ax2.set_ylabel("Freqency Hz")


        save1 = save_path+pathpart + changedir+specgram_path
        if not os.path.exists(save1):
            os.makedirs(save1)
        savefig(os.path.join(save1,str(Date) + "-" + str(Time) + "-Power_Spec1.png"))

        logging.info("Spectrogram path is " + save1)




        fig2, (ax4,ax6) = plt.subplots(nrows=2)
        final_fft = []
        fft = s.Full_FFT(filter1,window2)
        for i in range(0,len(fft)):
            final_fft.append(np.absolute(fft[i]))



        freqs = []
        for i in range(0,len(final_fft)):
            freqs.append(i*Sample_Rate/float(len(final_fft)))

        ax4.plot(freqs, final_fft)

        new_fft = []
        new = s.Full_FFT(filter2,window2)
        for i in range(0,len(new)):
            new_fft.append(np.absolute(new[i]))
        ax6.plot(freqs,new_fft)


        save2 = save_path+pathpart+ changedir + freq_path
        logging.info("Frequency path is " + save2)
        if not os.path.exists(save2):
            os.makedirs(save2)

        savefig(os.path.join(save2,str(Date) + "-" + str(Time) + "-Freq.png"))

        ax4.set_title(str(Date) + "-" + str(Time) +" Channel 1")
        ax4.set_xlabel("Bins")
        ax4.set_ylabel("Power")


        ax6.set_title(str(Date) + "-" + str(Time) + " Channnel 2")
        ax6.set_xlabel("Bins")
        ax6.set_ylabel("Power")

        fig3, (ax7, ax9) = plt.subplots(nrows=2)
        ax7.plot(t,filter1)
        ax9.plot(t,filter2)

        save3 = save_path+pathpart + changedir +signal_path
        if not os.path.exists(save3):
            os.makedirs(save3)
        savefig(os.path.join(save3,str(Date) + "-" + str(Time) + "-Signal.png"))

        logging.info("Signal path is " + save3)

        fig1.clf()
        fig2.clf()
        fig3.clf()
        matplotlib.pyplot.clf()
        close('all')
        gc.collect()

这是解压代码

def unpackFile(path,fileName):
header = ""
startKey = ""
dataList = []
chan1 = []
chan2 = []
found = False
logging.info("Starting file " + fileName)
if not os.path.isfile(os.path.join(path,fileName)):
    logging.warning("could not find "+fileName)
    return None, None, None
try:
    contents = open(os.path.join(path,fileName),'rb')
except IOError:
    logging.warning(fileName + " Not found")
    return None, None, None



#looks for the closing bracket in the header of the file
filesize = os.path.getsize(os.path.join(path,fileName))
if filesize < 1000:
    logging.warning(fileName + " is below 1000 bytes")
    contents.close()
    contents = None
    return None, None, None
while found==False:
    char = contents.read(1)
    #print char
    header = header + char
    if char == "}":
        #Once the close bracket is found, the next 10 characters should be the start key
        startKey = contents.read(10)
        #header = header + startKey
        #print("found the }")
        found = True 
if startKey=="Data_Start":
    logging.info("Found start key for file "+fileName)
else:
    logging.warning("No start key found " + fileName + " is corrupted")
    contents.close()
    contents = None
    return None, None, None   
#Looks for the end key in the file 
try:
    logging.debug("Reading the data")
    data = contents.read(40000)
    #endKey = data[len(data)-10:len(data)]
    endKey = contents.read()
except IOError:
    logging.warning("IOE error trying to read the end key")
    endKey=""
    contents.close()
    contents = None
    return None, None, None

if endKey == "Data_Stop ":
    logging.debug("Found end key " )
else:
    logging.debug("No end key found in" +fileName)
#Unpacks the data from binary into signed ints
for i in range(0,len(data),2):
    value = data[i:i+2]
    if len(value)==2:
        number = struct.unpack('>h',data[i:i+2])
        #print number
        dataList.append(number[0])
    else:
        break
logging.debug("total points found is " + str(len(dataList)))
#Splits data into two channels
for j in range(0,len(dataList)):
    if j%2==0:
        chan2.append(dataList[j])
        #if dataList[j] != 0:
            #print("chan2 has a non 0 " + str(j))
    else:
        chan1.append(dataList[j])
#Checks to make sure both channels contain 10000 data points. If this is not true the file is curppted
if len(chan2)!=10000:
    logging.warning("Chanel 2 did not containg the right number of data points, " + fileName + " is corupted")
    contents.close()
    contents = None
    return None, None, None
if len(chan1)!=10000:
    logging.warning("Chanel 1 did not containg the right number of data points, " + fileName + " is corupted")
    contents.close()
    contents = None
    return None, None, None
contents.close()
contents = None

header = header[1:len(header)-1]
header_parts = header.split(',')
return header_parts,chan1,chan2

某处内存泄漏，我不知道在哪里。我试图让代码遍历目录，挑选出数据集，然后绘制它们。几分钟后，这会吃掉几 GB 的内存。有什么减少它们的技巧吗？

Answer 1

在像您这样的情况下，我更愿意将清理工作留给操作系统。因此，我将泄漏内存的代码作为一个单独的进程运行:

from multiprocessing import Process, Queue

def memory_leaking_code(arg1, q):
   """Your memory leaking code goes here"""
   print arg1
   q.put('data from memory_leaking_code()')

def main():
   q = Queue()
   p = Process(target=memory_leaking_code, args=('data to memory_leaking_code()', q))
   p.start()
   print q.get()
   if p.is_alive():
      p.terminate()

main()