{"id":"56cT6ShEu9","url":"https://pastebin.ca/56cT6ShEu9","raw_url":"https://raw.anybin.ca/56cT6ShEu9","visibility":"public","access":"public","created_at":1781420808408,"expires_at":1782025608408,"fetch_limit":null,"fetches_used":0,"reads_remaining":null,"size_bytes":9753,"syntax_hint":null,"title":null,"filename":null,"change_note":null,"cipher":null,"cipher_meta":null,"parent_id":null,"root_id":"56cT6ShEu9","version":1,"owner_id":null,"recipient_id":null,"body":"# -*- coding: UTF-8 -*-\nimport sys\nimport os\nos.environ['PYSPARK_PYTHON']='/home/zhangyu/anaconda3/bin/python3'\nfrom time import time\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom pyspark import SparkConf, SparkContext\nfrom pyspark.mllib.tree import DecisionTree\nfrom pyspark.mllib.regression import LabeledPoint\nimport numpy as np\nfrom pyspark.mllib.evaluation import RegressionMetrics\nimport math\ndef SetLogger( sc ):\n    logger = sc._jvm.org.apache.log4j\n    logger.LogManager.getLogger(\"org\"). setLevel( logger.Level.ERROR )\n    logger.LogManager.getLogger(\"akka\").setLevel( logger.Level.ERROR )\n    logger.LogManager.getRootLogger().setLevel(logger.Level.ERROR)\ndef extract_label(record):\n    label=(record[-1])\n    return float(label)\ndef convert_float(x):\n    return (0 if x==\"?\" else float(x))\ndef extract_features(record,featureEnd):\n    featureSeason=[convert_float(field)  for  field in record[2]]\n    features=[convert_float(field)  for  field in record[4: featureEnd-2]]\n    return  np.concatenate( (featureSeason, features))\ndef PrepareData(sc):\n    #----------------------1.导入并转换数据-------------\n    print(\"开始导入数据...\")\n    rawDataWithHeader = sc.textFile(\"hdfs://127.0.0.1:9000/hour.csv\")\n    header = rawDataWithHeader.first()\n    rawData = rawDataWithHeader.filter(lambda x:x !=header)\n    lines = rawData.map(lambda x: x.split(\",\"))\n    print (lines.first())\n    print(\"共计：\" + str(lines.count()) + \"项\")\n    #----------------------2.建立训练评估所需数据 RDD[LabeledPoint]-------------\n    labelpointRDD = lines.map(lambda r:LabeledPoint(\n                                                    extract_label(r),\n                                                    extract_features(r,len(r) - 1)))\n    print(labelpointRDD.first())\n    #----------------------3.以随机方式将数据分为3个部分并且返回-------------\n    (trainData, validationData, testData) = labelpointRDD.randomSplit([8, 1, 1])\n    print(\"将数据分trainData:\" + str(trainData.count()) +\n             \"   validationData:\" + str(validationData.count()) +\n             \"   testData:\" + str(testData.count()))\n    #print(labelpointRDD.first())\n    return (trainData, validationData, testData) #返回数据\ndef PredictData(sc,model):\n    #----------------------1.导入并转换数据-------------\n    print(\"开始导入数据...\")\n    rawDataWithHeader = sc.textFile(\"hdfs://127.0.0.1:9000/hour.csv\")\n    header = rawDataWithHeader.first()\n    rawData = rawDataWithHeader.filter(lambda x:x !=header)\n    lines = rawData.map(lambda x: x.split(\",\"))\n    #print (lines.first())\n    print(\"共计：\" + str(lines.count()) + \"项\")\n    #----------------------2.建立训练评估所需数据 LabeledPoint RDD-------------\n    labelpointRDD = lines.map(lambda r: LabeledPoint(\n                                                     extract_label(r),\n                                                     extract_features(r,len(r) - 1)))\n    #----------------------3.定义字典----------------\n    SeasonDict = { 1 : \"春\",  2 : \"夏\",  3 :\"秋\",  4 : \"冬\"   }\n    HoildayDict={  0 : \"非假日\", 1 : \"假日\"  }\n    WeekDict = {0:\"一\",1:\"二\",2:\"三\",3:\"四\",4 :\"五\",5:\"六\",6:\"日\"}\n    WorkDayDict={ 1 : \"工作日\",  0 : \"非工作日\"  }\n    WeatherDict={ 1 : \"晴\",  2 : \"阴\",  3 : \"小雨\", 4 : \"大雨\" }\n    #----------------------4.进行预测并显示结果--------------\n    for lp in labelpointRDD.take(100):\n        predict = int(model.predict(lp.features))\n        label=lp.label\n        features=lp.features\n        result = (\"正确\" if  (label == predict) else \"错误\")\n        error = math.fabs(label - predict)\n        dataDesc=\"  特征: \"+SeasonDict[features[0]] +\"季,\"+\\\n                            str(features[1]) + \"月,\" +\\\n                            str(features[2]) +  \"时,\"+ \\\n                            HoildayDict[features[3]] +\",\"+\\\n                            \"星期\"+WeekDict[features[4]]+\",\"+ \\\n                            WorkDayDict[features[5]]+\",\"+\\\n                            WeatherDict[features[6]]+\",\"+\\\n                            str(features[7] * 41)+ \"度,\"+\\\n                            \"体感\" + str(features[8] * 50) + \"度,\" +\\\n                            \"湿度\" + str(features[9] * 100) + \",\"+\\\n                            \"风速\" + str(features[10] * 67) +\\\n                            \" ==> 预测结果:\" + str(predict )+\\\n                            \"  , 实际:\" + str(label) + result +\",  误差:\" + str(error)\n        print(dataDesc)\n\ndef evaluateModel(model, validationData):\n    score = model.predict(validationData.map(lambda p: p.features))\n    scoreAndLabels=score.zip(validationData.map(lambda p: p.label))\n    metrics = RegressionMetrics(scoreAndLabels)\n    RMSE=metrics.rootMeanSquaredError\n    return( RMSE)\n\ndef trainEvaluateModel(trainData,validationData,\n                                           impurityParm, maxDepthParm, maxBinsParm):\n    startTime = time()\n    model = DecisionTree.trainRegressor(trainData,\n                                  categoricalFeaturesInfo={}, \\\n                                  impurity=impurityParm,\n                                  maxDepth=maxDepthParm,\n                                  maxBins=maxBinsParm)\n    RMSE = evaluateModel(model, validationData)\n    duration = time() - startTime\n    print    (\"训练评估：使用参数\" + \\\n                \" impurityParm= %s\"%impurityParm+ \\\n                \"  maxDepthParm= %s\"%maxDepthParm+ \\\n                \"  maxBinsParm = %d.\"%maxBinsParm + \\\n                 \"  所需时间=%d\"%duration + \\\n                 \"  结果RMSE = %f \" % RMSE )\n    return (RMSE,duration, impurityParm, maxDepthParm, maxBinsParm,model)\ndef evalParameter(trainData, validationData, evaparm,impurityList, maxDepthList, maxBinsList):\n    metrics = [trainEvaluateModel(trainData, validationData,  impurity,maxdepth,  maxBins  )\n                            for impurity in impurityList\n                            for maxdepth in maxDepthList\n                            for maxBins in maxBinsList ]\n    if evaparm==\"impurity\":\n        IndexList=impurityList[:]\n    elif evaparm==\"maxDepth\":\n        IndexList=maxDepthList[:]\n    elif evaparm==\"maxBins\":\n        IndexList=maxBinsList[:]\n    df = pd.DataFrame(metrics,index=IndexList,\n                      columns=['RMSE', 'duration','impurityParm', 'maxDepthParm', 'maxBinsParm','model'])\n\n    showchart(df,evaparm,'RMSE','duration',0,200 )\n\n\ndef showchart(df,evalparm ,barData,lineData,yMin,yMax):\n    ax = df[barData].plot(kind='bar', title =evalparm,figsize=(10,6),legend=True, fontsize=12)\n    ax.set_xlabel(evalparm,fontsize=12)\n    ax.set_ylim([yMin,yMax])\n    ax.set_ylabel(barData,fontsize=12)\n    ax2 = ax.twinx()\n    ax2.plot(df[[lineData ]].values, linestyle='-', marker='o', linewidth=2.0,color='r')\n    plt.show()\n\ndef evalAllParameter(training_RDD, validation_RDD, impurityList, maxDepthList, maxBinsList):\n    metrics = [trainEvaluateModel(trainData, validationData,  impurity,maxdepth,  maxBins  )\n                        for impurity in impurityList\n                        for maxdepth in maxDepthList\n                        for maxBins in maxBinsList ]\n    Smetrics = sorted(metrics, key=lambda k: k[0])\n    bestParameter=Smetrics[0]\n\n    print(\"调校后最佳参数：impurity:\" + str(bestParameter[2]) +\n            \"  ,maxDepth:\" + str(bestParameter[3]) +\n            \"  ,maxBins:\" + str(bestParameter[4])   +\n            \"  ,结果RMSE = \" + str(bestParameter[0]))\n\n    return bestParameter[5]\ndef  parametersEval(training_RDD, validation_RDD):\n\n    print(\"----- 评估maxDepth参数使用 ---------\")\n    evalParameter(training_RDD, validation_RDD,\"maxDepth\",\n                              impurityList=[\"variance\"],\n                              maxDepthList =[3, 5, 10, 15, 20, 25]  ,\n                              maxBinsList=[10])\n    print(\"----- 评估maxBins参数使用 ---------\")\n    evalParameter(training_RDD, validation_RDD,\"maxBins\",\n                              impurityList=[\"variance\"],\n                              maxDepthList=[10],\n                              maxBinsList=[3, 5, 10, 50, 100, 200 ])\ndef CreateSparkContext():\n    sparkConf = SparkConf()                                            \\\n                         .setAppName(\"RunDecisionTreeRegression\")           \\\n                         .set(\"spark.ui.showConsoleProgress\", \"false\")\n    sc = SparkContext(conf = sparkConf)\n    print (\"master=\"+sc.master)\n    SetLogger(sc)\n    return (sc)\nif __name__ == \"__main__\":\n    print(\"RunDecisionTreeRegression\")\n    sc=CreateSparkContext()\n    print(\"==========数据准备阶段===============\")\n    (trainData, validationData, testData) =PrepareData(sc)\n    trainData.persist(); validationData.persist(); testData.persist()\n    print(\"==========训练评估阶段===============\")\n    (AUC,duration, impurityParm, maxDepthParm, maxBinsParm,model)= \\\n             trainEvaluateModel(trainData, validationData, \"variance\", 10, 100)\n    if (len(sys.argv) == 2) and (sys.argv[1]==\"-e\"):\n        parametersEval(trainData, validationData)\n    elif   (len(sys.argv) == 2) and (sys.argv[1]==\"-a\"):\n        print(\"-----所有参数训练评估找出最好的参数组合---------\")\n        model=evalAllParameter(trainData, validationData,\n                          [\"variance\"],\n                          [3, 5, 10, 15, 20, 25],\n                          [3, 5, 10, 50, 100, 200 ])\n    print(\"==========测试阶段===============\")\n    RMSE = evaluateModel(model, testData)\n    print(\"使用test Data测试最佳模型,结果 RMSE:\" + str(RMSE))\n    print(\"==========预测数据===============\")\n    PredictData(sc, model)\n    #print(model.toDebugString())"}