CODiS日報表下載整併

     1	#!/opt/anaconda3/envs/py27/bin/python
     2	#coding='utf8'
     3	from pandas import *
     4	from bs4 import BeautifulSoup
     5	import os,sys, subprocess, time
     6	
     7	

     8	def getarg():
     9	  """ read time period and station name from argument(std input)
    10	  rd_cwbDay.py -d 2017-12-31 """
    11	  import argparse
    12	  ap = argparse.ArgumentParser()
    13	  ap.add_argument("-d", "--DATE", required=True, type=str, help="yyyy-mm-dd")
    14	  args = vars(ap.parse_args())
    15	  return args['DATE']
    16	
    17	def is_date_valid(date):
    18	#    this_date = '%d/%d/%d' % (month, day, year)
    19	    try:
    20	        time.strptime(date, '%Y-%m-%d')
    21	    except ValueError:
    22	        return False
    23	    else:
    24	        return True
    25	

    26	date = getarg()
    27	if not  is_date_valid(date):sys.exit('not invalid:'+date)
    28	ymd = int(date.replace('-', ''))
    29	dir = '/Users/Data/cwb/e-service/read_web/'
    30	dir1 = dir + '../' + date[:4] + '/'
    31	fnameO=dir1 + 'cwb' + str(ymd) + '.csv'
    32	#if os.path.isfile(fnameO):sys.exit('file_exit:'+date)
    33	#touch and open for keeping from writing by others
    34	os.system('touch '+fnameO)
    35	ftext=open(fnameO,'w')
    36	

    37	#batcmd="date +%H"
    38	#ymdh = int(subprocess.check_output(batcmd, shell=True)[:-1])
    39	#dayt=0
    40	#if ymdh<19 and ymdh>=8:dayt=10
    41	mal = ['T', 'V', '/', 'X','&amp;']
    42	dfS = read_csv(dir + 'stats_tab.csv')
    43	h0 = 'https://e-service.cwb.gov.tw/HistoryDataQuery/'
    44	h1 = 'DayDataController.do\?command=viewMain\&station\='
    45	h2 = '\&stname\='
    46	h3 = '\&datepicker\='
    47	

    48	ib = 0
    49	for ii in range(ib, len(dfS)):
    50	  orig = h1 + dfS.loc[ii, 'stno'] + h2 + dfS.loc[ii, 'url_nam25'] + h3 + date
    51	  print h0 + orig + ' -O cwbDay.html'
    52	  os.system('curl ' + h0 + orig + ' -o cwbDay.html')
    53	  fname = 'cwbDay.html'
    54	  fn = open(fname, 'r')
    55	  soup = BeautifulSoup(fn, 'html.parser')
    56	

    57	  if ii == ib:
    58	    col_tr = soup.find_all("tr", class_="third_tr")
    59	    col_th = col_tr[0].find_all('th')
    60	    col = ['stno_name'] + \
    61	        [str(col_th[i]).split('>')[1].split('<')[0] for i in range(len(col_th))]
    62	    df = DataFrame({i: [] for i in col})
    63	    df.columns = col
    64	

    65	  dfi = DataFrame({i: [] for i in col})
    66	  dfi.columns = col
    67	  tr = soup.find_all('tr')
    68	  if len(tr) < 28: continue
    69	  stno_name = str(tr[0].find_all('td')[1]).split('>')[1].split('<')[0] \
    70	    .replace('\xc2', '').replace('\xa0', '').split(':')[1]
    71	  ymd = str(tr[0].find_all('td')[4]).split('>')[1].split('<')[0] \
    72	    .replace('\xc2', '').replace('\xa0', '').split(':')[1]
    73	  ymd = int(ymd.replace('-', ''))

    74	  for i in range(4, len(tr)):
    75	    a = tr[i].find_all('td')
    76	    col_val = []
    77	    for j in range(len(a)):
    78	      val = str(a[j]).split('>')[1].split('<')[0].replace('\xc2', '').replace('\xa0', '')
    79	      if len(val) == 0:
    80	        val = 0.
    81	      else:
    82	        if '.' not in val and val not in mal:
    83	          val = float(val)
    84	        elif val in mal:
    85	          val = 0.
    86	      if val == '...': val = ''
    87	      col_val.append(val)
    88	    col_val[0] = int(ymd * 100 + col_val[0])
    89	    col_val = [stno_name] + col_val
    90	    dfi.loc[i - 4, :] = col_val
    91	

    92	  if ii == ib:
    93	    df = dfi
    94	  else:
    95	    df = df.append(dfi, ignore_index=True)
    96	#  sec=str(round(np.random.rand(1)[0]*dayt,2))
    97	#  os.system('sleep '+sec+'s')
    98	  #  ifname+=1
    99	df.stno_name=[i[:6] for i in df.stno_name]
   100	os.system('mkdir -p ' + dir1)
   101	df.set_index('stno_name').to_csv(ftext)
   102	ftext.close()
   103	#sec=str(round(np.random.rand(1)[0]*dayt*12,2))
   104	#os.system('sleep '+sec+'s')

* old
    *     56     col_tr = soup.find_all("tr", class_="second_tr")
    *     57     col_th = col_tr[1].find_all('th')
* new
    *     56     col_tr = soup.find_all("tr", class_="third_tr")
    *     57     col_th = col_tr[0].find_all('th')