python - For 循环花费的时间太长

Question

编辑:如果这不是问题的话，这是循环上方的代码，可以获取更多有用的答案。

import os
import pandas as pd 
import numpy 
import csv 
from math import *
ParcelSize = 50
UARFCN = 3087 

y= r"C:\Users\Heba R\Desktop\GP\Pilot1.csv"
x= r"C:\Users\Heba R\Desktop\GP\Cell.csv"

scanner_File = pd.read_csv(y) 
Cell_file = pd.read_csv(x)
Cells = Cell_file[['Cell', 'Lat', 'Lon', 'SC', 'UARFCN', 'ANT_DIRECTION']]

scanner = scanner_File[
        ['Latitude', 'Longitude', 'PSC: Top #1 (UARFCN #01)', 'Sc Aggr Ec (dBm): Top #1 (UARFCN #01)',
         'Sc Aggr Ec/Io (dB): Top #1 (UARFCN #01)',
         'PSC: Top #2 (UARFCN #01)', 'Sc Aggr Ec (dBm): Top #2 (UARFCN #01)', 'Sc Aggr Ec/Io (dB): Top #2 (UARFCN #01)',
         'PSC: Top #3 (UARFCN #01)', 'Sc Aggr Ec (dBm): Top #3 (UARFCN #01)', 'Sc Aggr Ec/Io (dB): Top #3 (UARFCN #01)',
         'PSC: Top #4 (UARFCN #01)', 'Sc Aggr Ec (dBm): Top #4 (UARFCN #01)', 'Sc Aggr Ec/Io (dB): Top #4 (UARFCN #01)',
         'PSC: Top #5 (UARFCN #01)', 'Sc Aggr Ec (dBm): Top #5 (UARFCN #01)', 'Sc Aggr Ec/Io (dB): Top #5 (UARFCN #01)',
         'PSC: Top #6 (UARFCN #01)', 'Sc Aggr Ec (dBm): Top #6 (UARFCN #01)', 'Sc Aggr Ec/Io (dB): Top #6 (UARFCN #01)',
         'PSC: Top #7 (UARFCN #01)', 'Sc Aggr Ec (dBm): Top #7 (UARFCN #01)', 'Sc Aggr Ec/Io (dB): Top #7 (UARFCN #01)',
         'PSC: Top #8 (UARFCN #01)', 'Sc Aggr Ec (dBm): Top #8 (UARFCN #01)', 'Sc Aggr Ec/Io (dB): Top #8 (UARFCN #01)',
         'PSC: Top #9 (UARFCN #01)', 'Sc Aggr Ec (dBm): Top #9 (UARFCN #01)',
         'Sc Aggr Ec/Io (dB): Top #9 (UARFCN #01)']]
scanner_size = scanner.shape[0] 
cells_size = Cells.shape[0]

def CalcDistanceM(lat1, lon1, lat2, lon2): 
        lat1, lon1, lat2, lon2 = map(radians, [lat1, lon1, lat2, lon2]) #convert decimal to rad 
        #haversine formula to calculate two points great circle distance on earth
        dlon = lon2 - lon1
        dlat = lat2 - lat1
        a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2
        c = 2 * atan2(sqrt(a), sqrt(1 - a))
        distance = 6371 * c * 1000 #radius of earth in km =6371
        return distance

def fn_CalcParcelID(Pos, ParcelUnitSize):
        if (Pos == 500):  # null parcel
            Result = int(50000000)
        elif (Pos < 0):
            Result = int(Pos * 100000) - ParcelUnitSize + (int(Pos * 100000) % ParcelUnitSize)
        else:
            Result = int(Pos * 100000) - (int(Pos * 100000) % ParcelUnitSize)
        return int(Result)

A1=pd.DataFrame(columns=['Latitude','Longitude','PSC','EcNo','RSCP'])
A2=pd.DataFrame(columns=['Latitude','Longitude','PSC','EcNo','RSCP'])
A3=pd.DataFrame(columns=['Latitude','Longitude','PSC','EcNo','RSCP'])
A4=pd.DataFrame(columns=['Latitude','Longitude','PSC','EcNo','RSCP'])
A5=pd.DataFrame(columns=['Latitude','Longitude','PSC','EcNo','RSCP'])
A6=pd.DataFrame(columns=['Latitude','Longitude','PSC','EcNo','RSCP'])
A7=pd.DataFrame(columns=['Latitude','Longitude','PSC','EcNo','RSCP'])
A8=pd.DataFrame(columns=['Latitude','Longitude','PSC','EcNo','RSCP'])
A9=pd.DataFrame(columns=['Latitude','Longitude','PSC','EcNo','RSCP'])
for i in range (scanner_size):
       #if isnan(scanner['PSC: Top #1 (UARFCN #01)'][i]) == False:
       if (scanner['PSC: Top #1 (UARFCN #01)'][i]) != -1 :
          A1 = A1.append({ 'Latitude': scanner['Latitude'][i], 'Longitude': scanner['Longitude'][i],
                            'PSC': scanner['PSC: Top #1 (UARFCN #01)'][i],'EcNo': scanner['Sc Aggr Ec/Io (dB): Top #1 (UARFCN #01)'][i],'RSCP': scanner['Sc Aggr Ec (dBm): Top #1 (UARFCN #01)'][i]}, ignore_index=True)

       if (scanner['PSC: Top #2 (UARFCN #01)'][i]) !=-1:
           A2 = A2.append({'Latitude': scanner['Latitude'][i], 'Longitude': scanner['Longitude'][i],
                           'PSC': scanner['PSC: Top #2 (UARFCN #01)'][i],
                           'EcNo': scanner['Sc Aggr Ec/Io (dB): Top #2 (UARFCN #01)'][i],
                           'RSCP': scanner['Sc Aggr Ec (dBm): Top #2 (UARFCN #01)'][i]}, ignore_index=True)
       if (scanner['PSC: Top #3 (UARFCN #01)'][i]) != -1:
           A3 = A3.append({'Latitude': scanner['Latitude'][i], 'Longitude': scanner['Longitude'][i],
                           'PSC': scanner['PSC: Top #3 (UARFCN #01)'][i],
                           'EcNo': scanner['Sc Aggr Ec/Io (dB): Top #3 (UARFCN #01)'][i],
                           'RSCP': scanner['Sc Aggr Ec (dBm): Top #3 (UARFCN #01)'][i]}, ignore_index=True)
       if  (scanner['PSC: Top #4 (UARFCN #01)'][i]) != -1:
           A4 = A4.append({'Latitude': scanner['Latitude'][i], 'Longitude': scanner['Longitude'][i],
                           'PSC': scanner['PSC: Top #4 (UARFCN #01)'][i],
                           'EcNo': scanner['Sc Aggr Ec/Io (dB): Top #4 (UARFCN #01)'][i],
                           'RSCP': scanner['Sc Aggr Ec (dBm): Top #4 (UARFCN #01)'][i]}, ignore_index=True)
       if  (scanner['PSC: Top #5 (UARFCN #01)'][i]) != -1:
           A5 = A5.append({'Latitude': scanner['Latitude'][i], 'Longitude': scanner['Longitude'][i],
                           'PSC': scanner['PSC: Top #5 (UARFCN #01)'][i],
                           'EcNo': scanner['Sc Aggr Ec/Io (dB): Top #5 (UARFCN #01)'][i],
                           'RSCP': scanner['Sc Aggr Ec (dBm): Top #5 (UARFCN #01)'][i]}, ignore_index=True)
       if  (scanner['PSC: Top #6 (UARFCN #01)'][i]) != -1:
           A6 = A6.append({'Latitude': scanner['Latitude'][i], 'Longitude': scanner['Longitude'][i],
                           'PSC': scanner['PSC: Top #6 (UARFCN #01)'][i],
                           'EcNo': scanner['Sc Aggr Ec/Io (dB): Top #6 (UARFCN #01)'][i],
                           'RSCP': scanner['Sc Aggr Ec (dBm): Top #6 (UARFCN #01)'][i]}, ignore_index=True)
       if  (scanner['PSC: Top #7 (UARFCN #01)'][i]) != -1:
           A7 = A7.append({'Latitude': scanner['Latitude'][i], 'Longitude': scanner['Longitude'][i],
                           'PSC': scanner['PSC: Top #7 (UARFCN #01)'][i],
                           'EcNo': scanner['Sc Aggr Ec/Io (dB): Top #7 (UARFCN #01)'][i],
                           'RSCP': scanner['Sc Aggr Ec (dBm): Top #7 (UARFCN #01)'][i]}, ignore_index=True)
       if  (scanner['PSC: Top #8 (UARFCN #01)'][i]) != -1:
           A8 = A8.append({'Latitude': scanner['Latitude'][i], 'Longitude': scanner['Longitude'][i],
                           'PSC': scanner['PSC: Top #8 (UARFCN #01)'][i],
                           'EcNo': scanner['Sc Aggr Ec/Io (dB): Top #8 (UARFCN #01)'][i],
                           'RSCP': scanner['Sc Aggr Ec (dBm): Top #8 (UARFCN #01)'][i]}, ignore_index=True)
       if  (scanner['PSC: Top #9 (UARFCN #01)'][i]) != -1:
           A9 = A9.append({'Latitude': scanner['Latitude'][i], 'Longitude': scanner['Longitude'][i],
                           'PSC': scanner['PSC: Top #9 (UARFCN #01)'][i],
                           'EcNo': scanner['Sc Aggr Ec/Io (dB): Top #9 (UARFCN #01)'][i],
                           'RSCP': scanner['Sc Aggr Ec (dBm): Top #9 (UARFCN #01)'][i]}, ignore_index=True)
A=pd.concat([A1,A2,A3,A4,A5,A6,A7,A8,A9],sort=False)
A = A[~A[['Latitude','Longitude','PSC','EcNo','RSCP']].apply(frozenset, axis=1).duplicated()] #~ is bitwise not frozenset elem remain unchanged after creation
A.to_csv('table_data_pilot.csv',index=True)
A = pd.read_csv('table_data_pilot.csv')
#A=A.iloc[:50,:].reset_index()
A_size = A.shape[0]

for i in range(A_size):
      j = i +1
      for j in range (A_size):
         dLat=A['Latitude'][i] - A['Latitude'][j]
         dLon=A['Longitude'][i] - A['Longitude'][j]
         if abs(dLat) < 0.00045 and abs(dLon) < 0.00045:
           distance = CalcDistanceM(A['Latitude'][j], A['Longitude'][j],
                                                A['Latitude'][i],
                                                A['Longitude'][i])
           print (distance)

B1 = pd.DataFrame(columns=['Lat','Lon','UARFCN','PSC','SC_Avg_EcNo','SC_Avg_RSCP'])

首先，我刚刚开始接触Python，所以我没有太多的知识。我试图搜索类似的问题，但找不到合适的解决方案。我使用以下代码:

for i in range(A_size):
      x1=float(fn_CalcParcelID(A['Latitude'][i], ParcelSize) )/ 100000
      x2=float(fn_CalcParcelID(A['Longitude'][i], ParcelSize) ) / 100000
      B1 = B1.append({'Lat': x1, 'Lon': x2,
                            'PSC': A ['PSC'][i],
                            'UARFCN':UARFCN,
                            'SC_Avg_EcNo':A['EcNo'][i],
                            'SC_Avg_RSCP': A['RSCP'][i]

                            }, ignore_index=True)
B1.to_csv('B1.csv')

循环的目的是计算新的纬度和经度，然后创建一个新的 csv 文件。A 是一个 csv 文件，大约有 23000 行和 42 列

Answer 1

一般来说，您应该尽可能避免使用 for 循环迭代 Pandas DataFrame。

关于 Iteration 的 Pandas 文档说:

Warning

Iterating through pandas objects is generally slow. In many cases, iterating manually over the rows is not needed and can be avoided with one of the following approaches:

• Look for a vectorized solution: many operations can be performed using built-in methods or NumPy functions, (boolean) indexing, …

• When you have a function that cannot work on the full DataFrame/Series at once, it is better to use apply() instead of iterating over the values. See the docs on function application.

此外，使用 append() 在循环内向 DataFrame 添加新行是相当有问题的。

关于 Concat 的文档解释一下:

Adding a column to a DataFrame is relatively fast. However, adding a row requires a copy, and may be expensive. We recommend passing a pre-built list of records to the DataFrame constructor instead of building a DataFrame by iteratively appending records to it. See Appending to dataframe for more.

如果在循环中执行此操作，则循环的每次迭代都会将所有数据从 DataFrame 复制到新的 DataFrame，只是为了添加一行。此外，此操作每次都会变得更加昂贵，因为 DataFrame 不断增长，并且每次都会有更多数据需要复制。

<小时/>

在您的具体情况下，您可以轻松避免大部分情况，方法是将 A 作为一个整体处理，生成要附加到 B1 的所有行，然后执行单个 em> append() 操作，意味着只需复制 B1 一次即可。

把它们放在一起:

rows_to_add = pd.DataFrame({
    'Lat': A['Latitude'].apply(
        lambda x: fn_CalcParcelID(x, ParcelSize) / 100000.0
    ),
    'Lon': A['Longitude'].apply(
        lambda x: fn_CalcParcelID(x, ParcelSize) / 100000.0
    ),
    'PSC': A['PSC'],
    'UARFCN': UARFCN,
    'SC_Avg_EcNo': A['EcNo'],
    'SC_Avg_RSCP': A['RSCP'],
})
B1 = B1.append(rows_to_add, ignore_index=True)

这应该可以让您从运行几分钟而看不到结束的情况到在几秒钟内完成此操作。

您可以通过使用矢量化运算实现 fn_CalcParcelID() 来进一步优化它。 (很难告诉我们如何做到这一点，因为您没有向我们展示该函数的实现。)但是第一个优化可能就是您所需要的。如果您觉得值得，请提出有关矢量化 fn_CalcParcelID() 的新问题。

<小时/>

更新:您的代码第一部分确实有一个相同问题的版本，您在其中循环遍历 scanner CSV 文件并将其重新组织为A1 至 A9。 (每个 for 循环内都有一个 A1 = A1.append(...)，因此循环中也有附加内容!)

您可以通过以下方式解决这个问题:

A1_rows = scanner[scanner['PSC: Top #1 (UARFCN #01)'] != -1]
A1 = pd.DataFrame({
    'Latitude': A1_rows['Latitude'],
    'Longitude': A1_rows['Longitude'],
    'PSC': A1_rows['PSC: Top #1 (UARFCN #01)'],
    'EcNo': A1_rows['Sc Aggr Ec/Io (dB): Top #1 (UARFCN #01)'],
    'RSCP': A1_rows['Sc Aggr Ec (dBm): Top #1 (UARFCN #01)'],
})

其他 8 个类似的 DataFrame 也类似。