from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV
import pandas as pd


def knncls():
    '''
    k-近邻算法预测用户入住位置
    :return: None
    '''
    # 读取数据
    data = pd.read_csv("./train.csv")
    # print(data.head(10))

    # 特征工程
    # 1.通过设定xy范围缩小数据
    data = data.query("x > 1.0 & x < 1.25 & y > 2.5 & y < 2.75")  # query相当于一个查询语句，参数为查询条件

    # 2.处理时间戳数据
    time_value = pd.to_datetime(data['time'], unit='s')  # 将时间戳转换为年-月-日 时-分-秒格式
    # print(time_value)
    '''
    timevalue的值为：
    0          1970-01-06 10:45:02
    1          1970-01-03 03:49:15
    2          1970-01-04 17:37:28
    3          1970-01-09 03:43:07
    '''
    # 把日期转换为字典格式
    time_value = pd.DatetimeIndex(time_value)
    # 3.添加构造一些特征
    data['day'] = time_value.day
    data['hour'] = time_value.hour
    data['weekday'] = time_value.weekday
    # 4.把时间戳特征删除,sklearn中axis为1表示列
    data = data.drop(['time'], axis=1)

    # 5.只保留入住人数大于3的目标位置,生成新的data
    place_count = data.groupby("place_id").count()  # 按照'place_id'进行分组,然后统计次数。groupby使用参考：https://blog.csdn.net/m0_37870649/article/details/80979809
    # print(place_count)
    '''
    此时print的place_count如下：
    此时place_id为索引，而原来的特征值都变为了对应索引的place_id出现了多少次
    place_id    row_id    x    y  accuracy  day  hour  weekday                                        
    1000015801      78   78   78        78   78    78       78
    1000017288      95   95   95        95   95    95       95
    1000025138     563  563  563       563  563   563      563
    1000052096     961  961  961       961  961   961      961
    '''
    place_count_r = place_count[
        place_count.row_id > 3].reset_index()  # 筛选place_count中row_id大于3的，然后通过reset_index()把place_id重新设为特征而不是索引
    train_data = data[data["place_id"].isin(place_count_r["place_id"])]     #通过对比筛选

    # 提取特征值和目标值
    x = train_data.drop(["place_id", "row_id"], axis=1)

    y = train_data["place_id"]

    # 分割数据集
    #参数：特征值，目标值，测试集比例。返回值依次为：训练集特征值、测试集特征值，训练集目标值，测试集目标值
    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25)

    # 对训练集和测试集的特征值进行标准化
    std = StandardScaler()

    x_train = std.fit_transform(x_train)
    x_test = std.transform(x_test)

    # 实例化knn估计器
    knn = KNeighborsClassifier()    # n_neighbors，默认5

    #knn.fit(x_train, y_train)   #参数：训练集特征值、训练集目标值
    # 预测结果
    #y_predict = knn.predict(x_test) #参数：测试集
    # 打印准确率
    #print("准确率为:", knn.score(x_test, y_test))
    '''
    准确率为: 0.4732860520094563
    '''

    # 进阶思考：
    # 交叉验证与网格搜索对K-近邻算法调优
    param = {"n_neighbors": [i * 2 for i in range(1, 5)]}
    gc = GridSearchCV(knn, param_grid=param, cv=5)
    gc.fit(x_train, y_train)

    print("在测试集中的准确率：", gc.score(x_test, y_test))
    print("在交叉验证中验证集的最好结果：", gc.best_score_)
    print("使用的最好的模型：", gc.best_estimator_)
    '''
    在测试集中的准确率： 0.47848699763593383
    在交叉验证中验证集的最好结果： 0.47540983606557374
    使用的最好的模型： KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',
           metric_params=None, n_jobs=None, n_neighbors=8, p=2,
           weights='uniform')
    '''
    return None
if __name__ == "__main__":
    knncls()

from sklearn.datasets import fetch_20newsgroups
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB


def naviebayes():
    '''
    朴素贝叶斯对新闻进行分类
    :return: None
    '''
    news = fetch_20newsgroups(subset='all')

    # 进行分割
    x_train, x_test, y_train, y_test = train_test_split(news.data, news.target, test_size=0.25)

    # 对数据集进行特征抽取,用tf-idf方法
    tf = TfidfVectorizer()

    # 以训练集中的词的列表进行每篇文章重要性统计
    x_train = tf.fit_transform(x_train)  # 此时x_train为sparse矩阵，大概内容就是记录着:(行，列) 对应数值
    print(tf.get_feature_names())
    '''
    特征名，也就是每个单词
    'davidst', 'davidstern', 'davidw', 'davie', 'daviel', 'davies', 'davinci', 'davis', 'davison',
    '''

    x_test = tf.transform(x_test)

    # 进行朴素贝叶斯算法的预测
    mlt = MultinomialNB(alpha=1.0)
    print(x_train.toarray())
    '''
    大多数是0，因为一篇文章不是所有的词都有，每一行代表一篇文章，列就是上面的特征名，即单词，行列对应值为TF-IDF值
    [[0. 0. 0. ... 0. 0. 0.]
    [0. 0. 0. ... 0. 0. 0.]
    ..................
    [0. 0. 0. ... 0. 0. 0.]]
    '''

    mlt.fit(x_train, y_train)

    y_predict = mlt.predict(x_test)
    print("预测的文章类别为：", y_predict)
    '''
    预测的文章类别为： [ 7 16  0 ...  4  8  1]
    '''

    # 得出准确率
    print("准确率为：", mlt.score(x_test, y_test))
    '''
    准确率为： 0.8293718166383701
    '''
    return None


if __name__ == "__main__":
    naviebayes()

import pandas as pd
from sklearn.feature_extraction import DictVectorizer
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier,export_graphviz

def decision():
    '''
    决策树对泰坦尼克号乘客进行预测生死
    :return:None
    '''
    # 获取数据
    titan = pd.read_csv("http://biostat.mc.vanderbilt.edu/wiki/pub/Main/DataSets/titanic.txt")

    # 处理数据，找出特征值和目标值,这里以三个为例
    x = titan[['pclass', 'age', 'sex']]
    print(x)
    '''
         pclass      age     sex
    0       1st  29.0000  female
    1       1st   2.0000  female
    2       1st  30.0000    male
    3       1st  25.0000  female
    '''
    y = titan['survived']

    # 缺失值处理
    x['age'].fillna(x['age'].mean(), inplace=True)

    # 分割数据集到训练集和测试集
    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25)

    # 进行特征工程，当特征是类别的时候，要进行ont-hot编码
    dict = DictVectorizer(sparse=False)

    x_train = dict.fit_transform(x_train.to_dict(orient="records"))
    print(dict.get_feature_names())
    '''
    ['age', 'pclass=1st', 'pclass=2nd', 'pclass=3rd', 'sex=female', 'sex=male']
    '''
    print(x_train)
    '''
    刚好与上面对应
    [[29.         1.          0.          0.          1.          0.        ]
    [ 2.          1.          0.          0.          1.          0.        ]
    [30.          1.          0.          0.          0.          1.        ]
    '''
    x_test = dict.transform(x_test.to_dict(orient="records"))

    # 用决策树估计器进行预测
    dec = DecisionTreeClassifier()  # max_depth是个超参数，值可以影响结果，也决定了决策树的深度
    dec.fit(x_train,y_train)

    #预测准确率
    print("DecisionTreeClassifier无参数时，预测的准确率为：",dec.score(x_test,y_test))
    '''
    DecisionTreeClassifier无参数时，预测的准确率为： 0.8206686930091185
    '''

    #导出决策树的结构，截图在之后给出
    export_graphviz(dec,out_file="./tree.dot",feature_names=['age', 'pclass=1st', 'pclass=2nd', 'pclass=3rd', 'sex=女性', 'sex=男性'])
    return None


if __name__ == "__main__":
    decision()

import pandas as pd
from sklearn.feature_extraction import DictVectorizer
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier

def decision():
    '''
    决策树对泰坦尼克号乘客进行预测生死
    :return:None
    '''
    # 获取数据
    titan = pd.read_csv("http://biostat.mc.vanderbilt.edu/wiki/pub/Main/DataSets/titanic.txt")

    # 处理数据，找出特征值和目标值,这里以三个为例
    x = titan[['pclass', 'age', 'sex']]
    print(x)
    '''
         pclass      age     sex
    0       1st  29.0000  female
    1       1st   2.0000  female
    2       1st  30.0000    male
    3       1st  25.0000  female
    '''
    y = titan['survived']

    # 缺失值处理
    x['age'].fillna(x['age'].mean(), inplace=True)

    # 分割数据集到训练集和测试集
    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25)

    # 进行特征工程，当特征是类别的时候，要进行ont-hot编码
    dict = DictVectorizer(sparse=False)

    x_train = dict.fit_transform(x_train.to_dict(orient="records"))
    print(dict.get_feature_names())
    '''
    ['age', 'pclass=1st', 'pclass=2nd', 'pclass=3rd', 'sex=female', 'sex=male']
    '''
    print(x_train)
    '''
    刚好与上面对应
    [[29.         1.          0.          0.          1.          0.        ]
    [ 2.          1.          0.          0.          1.          0.        ]
    [30.          1.          0.          0.          0.          1.        ]
    '''
    x_test = dict.transform(x_test.to_dict(orient="records"))

    # 随机森林估计器进行预测，超参数调优(这里以两个参数为例）
    rf = RandomForestClassifier()

    #网格搜索与交叉验证寻找最佳参数
    param = {"n_estimators":[10,20,100,120,200,300,500,800],"max_depth":[5,8,15,20,30]}
    gc = GridSearchCV(rf,param_grid=param,cv=2)
    gc.fit(x_train,y_train)
    print("准确率为：",gc.score(x_test,y_test))
    '''
    准确率为： 0.8267477203647416
    '''
    print("选择的参数模型：",gc.best_params_)
    '''
    选择的参数模型： {'max_depth': 5, 'n_estimators': 10}
    '''
    return None


if __name__ == "__main__":
    decision()

from sklearn.datasets import load_boston
from sklearn.linear_model import LinearRegression, SGDRegressor
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error


def mylinear():
    '''
    线性回归预测房子价格
    :return: None
    '''
    # 获取数据
    lb = load_boston()
    # 分割数据集到训练集和测试集
    x_train, x_test, y_train, y_test = train_test_split(lb.data, lb.target, test_size=0.25)

    # 进行标准化
    # 这里特征值和目标值都要进行
    std_x = StandardScaler()  # 对特征值进行标准化
    x_train = std_x.fit_transform(x_train)
    x_test = std_x.transform(x_test)

    std_y = StandardScaler()  # 对目标值进行标准化
    y_train = std_y.fit_transform(y_train.reshape(-1, 1))  # 要求是二维
    y_test = std_y.fit_transform(y_test.reshape(-1, 1))

    # estimator预测

    # 正规方程求解方式预测
    lr = LinearRegression()
    lr.fit(x_train, y_train)
    print("回归系数为：", lr.coef_)
    ''''
    回归系数为： [[-0.13482789  0.14684301  0.01549439  0.03522313 -0.21053828  0.32880086
    -0.00599198 -0.39363529  0.25896495 -0.2026098  -0.21658332  0.09938895
    -0.38005182]]
    '''
    # 预测测试集的房子价格
    y_lr_predict = std_y.inverse_transform(lr.predict(x_test))  # 此时的值还是标准化后的,所以要inverse

    print("测试集里每个房子的预测价格：", y_lr_predict)
    '''
    测试集里每个房子的预测价格： [[-5.23859608]
    [39.61950876]
    [35.86414606]
    ......
    [20.31808398]
    [17.36664138]]
    '''
    print("正规方程的均方差：", mean_squared_error(std_y.inverse_transform(y_test), y_lr_predict))
    '''
    正规方程的均方差： 24.203056713653485
    '''
    # 梯度下降进行房价预测
    sgd = SGDRegressor()  # 这里可以指定学习率
    sgd.fit(x_train, y_train)
    print("SGD预测回归系数为：", sgd.coef_)
    ''''
    SGD预测回归系数为： [-0.08189344  0.07032661 -0.03413881  0.10527339 -0.09947185  0.33781714
    -0.04008106 -0.21279579  0.06362954 -0.05064323 -0.19061834  0.10570017
    -0.36071628]
    '''
    # 预测测试集的房子价格
    y_sgd_predict = std_y.inverse_transform(sgd.predict(x_test))  # 此时的值还是标准化后的,所以要inverse

    print("测试集里每个房子的预测价格：", y_sgd_predict)
    '''
    测试集里每个房子的预测价格：[22.25491159 30.38773853 12.72306638 35.68389966 29.69842243 21.70338892
    15.23925266 27.85422604 24.72436984 31.05806439 16.91816538 28.40787834
    ......
    21.3051416  23.92421059 20.83558394 23.80267794 28.17431605 32.1886237
    25.89847536]
    '''

    print("梯度下降的均方差：", mean_squared_error(std_y.inverse_transform(y_test), y_sgd_predict))
    '''
    梯度下降的均方差： 24.303529166136546
    '''
    return None


if __name__ == "__main__":
    mylinear()

from sklearn.metrics import mean_squared_error,classification_report
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
import pandas as pd
import numpy as np

def logistic():
    '''
    逻辑回归做二分类进行癌症肿瘤是否是恶性的预测（根据属性特征）
    :return: None
    '''
    # 构造列标签名字,注：如果数据中没有标签，则pandas会默认把第一行数据作为特征名，这里数据是没有标签名的，所以这里手动添加
    column = ['Sample code number','Clump Thickness','Uniformity of Cell Size','Uniformity of Cell Shape','Marginal Adhesion','Single Epithelial Cell Size'
              ,'Bare Nuclei','Bland Chromatin','Normal Nucleoli','Mitoses','Class']
    data = pd.read_csv("http://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/breast-cancer-wisconsin.data"
                , names=column)

    #缺失值进行处理
    data = data.replace(to_replace='?',value=np.nan)
    data = data.dropna()

    # 数据分割
    x_train,x_test,y_train,y_test = train_test_split(data[column[1:10]],data[column[10]],test_size=0.25)

    #进行标准化
    std = StandardScaler()
    x_train = std.fit_transform(x_train)
    x_test = std.transform(x_test)

    #逻辑回归预测
    lg = LogisticRegression(penalty='l2',C=1.0)
    lg.fit(x_train,y_train)
    y_predict = lg.predict(x_test)
    print("计算的回归系数为：",lg.coef_)
    '''
    计算的回归系数为： [[1.07139763 0.920439   1.03352718 0.88427896 0.07783286 1.15006497
    0.94210002 0.59530658 0.81971895]]
    '''
    print("准确率为：",lg.score(x_test,y_test))
    '''
    准确率为： 0.9590643274853801
    '''
    print("召回率：",classification_report(y_test,y_predict,labels=[2,4],target_names=["良性","恶性"]))
    '''
    召回率：       precision    recall  f1-score   support

          良性       0.96      0.97      0.97       110
          恶性       0.95      0.93      0.94        61

     micro avg       0.96      0.96      0.96       171
     macro avg       0.96      0.95      0.96       171
  weighted avg       0.96      0.96      0.96       171
    '''
    return None


if __name__ == "__main__":
    logistic()

from numpy import *
from math import sqrt

def loadDataSet(fileName):      #general function to parse tab -delimited floats
    dataMat = []                #assume last column is target value
    fr = open(fileName)
    for line in fr.readlines():
        curLine = line.strip().split('\t')
        fltLine = map(float,curLine) #map all elements to float()
        dataMat.append(fltLine)
    return dataMat
\#计算欧氏距离
def distEclud(vecA, vecB):
    return sqrt(sum(power(vecA - vecB, 2))) #la.norm(vecA-vecB)

def randCent(dataSet, k):
    n = shape(dataSet)[1]
    centroids = mat(zeros((k,n)))#create centroid mat
    for j in range(n):#create random cluster centers, within bounds of each dimension
        minJ = min(dataSet[:,j]) 
        rangeJ = float(max(dataSet[:,j]) - minJ)
        centroids[:,j] = mat(minJ + rangeJ * random.rand(k,1))
    return centroids

\#dataSet样本点,k 簇的个数
\#disMeas距离量度，默认为欧几里得距离
\#createCent,初始点的选取
def kMeans(dataSet, k, distMeas=distEclud, createCent=randCent):
    m = shape(dataSet)[0] #样本数
    clusterAssment = mat(zeros((m,2))) #m*2的矩阵                   
    centroids = createCent(dataSet, k) #初始化k个中心
    clusterChanged = True             
    while clusterChanged:      #当聚类不再变化
        clusterChanged = False
        for i in range(m):
            minDist = inf; minIndex = -1
            for j in range(k): #找到最近的质心
                distJI = distMeas(centroids[j,:],dataSet[i,:])
                if distJI < minDist:
                    minDist = distJI; minIndex = j
            if clusterAssment[i,0] != minIndex: clusterChanged = True
            # 第1列为所属质心，第2列为距离
            clusterAssment[i,:] = minIndex,minDist**2
        print centroids

        # 更改质心位置
        for cent in range(k):
            ptsInClust = dataSet[nonzero(clusterAssment[:,0].A==cent)[0]]
            centroids[cent,:] = mean(ptsInClust, axis=0) 
    return centroids, clusterAssment

# coding=utf-8
# 折线图
from matplotlib import pyplot as plt
import matplotlib

# 注：matplotlib默认时不支持中文的，可通过rc修改
font = {'family': 'SimHei',
        'weight': 'bold',
        'size': '16'}
matplotlib.rc('font', **font)

x = range(2, 26, 2)
y = [15, 13, 12, 11, 15, 3, 20, 22, 14.5, 6.3, 21.2, 10]
# 要在一个图形上绘制多个曲线，可直接设置另一个列表然后通过plot来绘制
another_y = [10, 10, 22, 15, 13, 5, 6, 8, 11.5, 20, 21.2, 20]

# 设置图片大小
# plt.figure(figsize=(20, 8), dpi=80)

# 绘图，label为折线名字，需要在后面添加legend方法才能显示在图形上，还可设置线条颜色和style等
plt.plot(x, y, label="第一条")
# 绘制另一条曲线
plt.plot(x, another_y, label="第二条")

# 绘制x轴的刻度,根据列表参数的值展示刻度
# plt.xticks(x)
# plt.xticks(range(2,25))
# x轴刻度显示字符串, 需要构造一个和xticks第一个参数一一对应的字符串列表，这也可以解决xy轴刻度对不准问题，自定义在哪里显示刻度和显示什么
_xstring_labels = ["string{}".format(i) for i in x]
plt.xticks(x, _xstring_labels, rotation=90)  # rotation可以让x刻度旋转一定角度

# 展示y轴刻度
plt.yticks(range(min(y), max(y)))

# 添加图形和坐标轴的描述信息
plt.xlabel("x轴信息")
plt.ylabel("y轴信息")
plt.title("图形名字")

# 绘制网格,参数alpha可以设置透明度
plt.grid()

# 添加图例，loc参数可设置图例位置
plt.legend()

# 展示图形
plt.show()

# 保存图形,后缀为svg可以保存为矢量图
# plt.savefig("./testone.png")

# coding = utf-8
import numpy
import random

# numpy 创建数组
t1 = numpy.array([1, 2, 3])  # [1,2,3]
t2 = numpy.array(range(1, 6))  # [1,2,3,4,5]
t3 = numpy.arange(4, 10, 2)  # [4,6,8]

print(t3.dtype)  # int32 与平台有关，还可能是int64,float32等
t4 = numpy.arange(4, 10, 2, dtype="int64")  # 可以手动指定数值类型
# 调整数据类型
t5 = t4.astype("float32")
print(t5.dtype)

t6 = random.random()
t6 = numpy.round(t6, 2)  # round保留两位小数
print(t6)

'''
numpy读取本地CSV文件
api：numpy.loadtxt(fname, dtype=, comments='#', delimiter=None, converters=None, skiprows=0, usecols=None, unpack=False, ndmin=0)
fname :文件或字符串
dtype : 数据类型
comments :用于指示注释开头的字符，默认值为 : '#'.
delimiter :用于分隔值的字符，缺省值为任何空白字符，如空格 ，制表符  
converters : dict字典 ，用来定义将对应的列转换为浮点数的函数。列入：0列是日期字符串："converters={0:datestr2num}".convertes 同时也能够为丢失的数据设置缺省值："convertes={3:lambda s:float(s.strip() or 0)}"
skiprows:跳过开头的行数
usecols : 确定那几列被读取
unpack :是否进行转置
'''
# n1 = numpy.loadtxt("test.csv",delimiter=",",dtype="int") # t1= [ [0行0列 0行1列 0行2列] [1行0列 1行1列 1行2列] [2行0列 2行1列 2行2列] ]

n2 = numpy.array([
    [1, 2, 3, 4],
    [5, 6, 7, 8],
    [9, 10, 11, 12],
])
# 索引和切片
# print(n2[行,列]) ‘：’表所有
# 取一行
print(n2[1])  # [5 6 7 8]
# 取连续多行
print(n2[1:])  # [[5 6 7 8] [9 10 11 12]]
# 取不连续多行
print(n2[[0, 2]])  # [ [1 2 3 4] [9 10 11 12]]

# 取列
print(n2[:, 0])  # [1 5 9]
# 连续多列
print(n2[:, 2:])  # [ [3 4] [7 8 ] [11 12]]
# 不连续的列
print(n2[:, [0, 2]])  # [ [1 3][5 7][9 11] ]

# 取单行单列和多行多列
print(n2[[0, 1], [1, 2]])  # 注意，这里是 [2 7]
print(n2[1, 2])  # [7]
print(n2[0:2, 0:3])  # [[1 2 3][5 6 7]] ，这里的多行多列，‘:’的区间是[)

# 数据拼接
n3 = numpy.array([
    [1, 2, 3, 4],
    [5, 6, 7, 8]
])
n4 = numpy.array([
    [9, 10, 11, 12],
    [11, 12, 13, 14]
])
# 横向和竖向拼接
numpy.vstack((n3, n4))  # [[1 2 3 4] [5 6 7 8] [9 10 11 12] [...]]
numpy.hstack((n3, n4))  # [ [1 2 3 4 9 10 11 12] [5 6 7 8 11 12 13 14]]
# numpy数组可以支持 a b = b a语法来换行或者换列，如
n2[[1, 2], :] = n2[[2, 1], :]  # 行交换
n2[:, [0, 1]] = n2[:, [1, 0]]  # 列交换

# coding = 'utf-8'
import pandas as pd
import numpy as np

# Series相关

t1 = pd.Series([1, 2, 33, 44, 5])
t2 = pd.Series([1, 2, 3, 4, 5], index=list("abcde"))  # 此时 1的索引为a,2的索引为b,以此类推
print(t2.dtype)  # int64
temp_list = {"name": "q", "age": 26, "tel": "10086"}
t3 = pd.Series(temp_list)  # 此时索引为字典的键，值为字典对应键的值
# tip:对一个series重新指定索引后，如果能够对应上，就取其值，如果不能，则为Nan
print(t3.dtype)  # object

# Series的切片和索引
print(t3[:2])  # 取前3行
print(t3[["name", "age"]])  # 取name和age
print(t1[t1 > 2])  # 取值大于2的
print(t3.index)  # 获取索引列表，类型为Index，可以使用list(t3.index) 强制转换为列表
print(t3.values)  # 获取值的列表,类型为numpy.ndarry，其实就是一个数组
# tips：ndarry中很多方法在pandas中都可以用。但pandas中的where是不同的,可以用来查找符合条件的数值，而不符合的显示为指定的值，不指定则为Nan
print('*' * 20)

# DataFrame相关

p1 = pd.DataFrame(np.arange(12).reshape(3, 4))
print(p1)
# 带索引的创建,指定行索引和列索引
p2 = pd.DataFrame(np.arange(12).reshape(3, 4), index=list("abc"), columns=list("WXYZ"))
print(p2)
# 传入字典创建,键为列名,每一行为具体的值，值可以对应一个列表，列表内有多少数据就有多少行。也可以传入一个内容是字典的列表，有几个字典就有几行。没有的数据为NaN
d1 = {"name": ["q1", "q2"], "age": [25, 26], "tel": [10086, 10010]}
p3 = pd.DataFrame(d1)
print(p3)
d2 = [{"name": "q1", "age": 25, "tel": 10010}, {"name": "q2", "tel": 10000}, {"name": "q3", "age": 22, "tel": 10086}]
p4 = pd.DataFrame(d2)
print(p4)
# 前几行，tail为尾几行。info可以查看DataFrame的内存情况，多少行多少列占用多少内存等
p4.head(1)

# DataFrame取行取列，方括号内写数组，表示取行，写字符串，表示取列的索引
p5 = pd.DataFrame(np.arange(100).reshape(10, 10), index=["r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10"],
                  columns=["c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10"])
# 取前5行
print(p5[:5])
# 取固定列
print(p5["c2"])
# 通过pandas优化过的选择方式：
# loc通过标签索引行数据，loc中冒号两边的数据取的方式是闭合区间[]，而不是[)
# iloc通过位置获取行数据
print(p5.loc["r4", "c5"])  # 取r4行r5列
print(p5.loc["r4"])  # 取r4行
print(p5.loc[["r1", "r3"]])  # 取多行多列语法 如r1行和r3行
print(p5.iloc[:, 2])  # 取第2列,索引从0开始
print(p5.iloc[5:, 3:])  #取第6行以后和第3列以后的数据
print(p5[p5["c7"]>70])  #取c7列大于70的所有行

#缺失数据NaN的处理
p6 = pd.DataFrame(np.array([0,1,np.NaN,3,4,5,6,7,8,np.NaN,10,11]).reshape(3, 4), index=list("abc"), columns=list("WXYZ"))
print(p6)
print(pd.notnull(p6))   #p6中，不为NaN的值显示为true，为NaN的值显示为false
print(p6[pd.notnull(p6["X"])])  #选取p6的X列中的值不为NaN的所有行
print(p6.dropna(axis=0,how="any"))    #删除DataFrame中值为NaN的axis，axis=0为行，1为列。how的值为“any”表示这个轴只要有NaN就删，值为“all”表示所有值都为NaN才删
#填充NaN为指定数据
print(p6.fillna(0))     # 将NaN填充为0
# 不过为了数据完整性，一般不会填充为0或者随便填一个数值，而是会填充为均值,Pandas计算均值的时候不会算上NaN值
print(p6.mean())    #此时mean算的均值是每一列的均值
print(p6.fillna(p6.mean()))
print(p6["X"].fillna(p6["X"].mean()))   #只填充某一列的NaN值，如果要改变原数据,可以前面添加 p6["X"] =

# pandas读取外部数据
# csv文件
# result = pd.read_csv("./test.csv")
# 读取sql，传入sql语句和connection
# pd.read_sql(sql_sentence,connection)

# coding=utf-8
import requests
from lxml import html
import threading
from queue import Queue
import os

class QuibaiSpdier:
    def __init__(self):
        self.url_temp = "https://www.qiushibaike.com/hot/page/{}/"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36"}
        self.url_queue = Queue()  # 需要请求的url地址
        self.html_queue = Queue()
        self.content_queue = Queue()

    def get_url_list(self):
        # return [self.url_temp.format(i) for i in range(1, 14)]
        for i in range(1, 14):
            self.url_queue.put(self.url_temp.format(i))

    def parse_url(self):
        while True:
            url = self.url_queue.get()
            print("request:" + url)
            response = requests.get(url, headers=self.headers)
            # return response.content.decode()
            self.html_queue.put(response.content.decode())
            self.url_queue.task_done()

    def get_content_list(self):  # 提取数据
        while True:
            html_str = self.html_queue.get()
            html_elements = html.etree.HTML(html_str)
            div_list = html_elements.xpath("//div[@id='content-left']/div")
            content_list = []
            for div in div_list:
                item = {}
                item["content"] = div.xpath(".//div[@class='content']/span/text()")
                item["author_gender"] = div.xpath(".//div[contains(@class,'articleGender')]/@class")
                item["author_gender"] = item["author_gender"][0].split(" ")[-1].replace("Icon", "") if len(
                    item["author_gender"]) > 0 else None
                item["content_img"] = div.xpath(".//div[@class='thumb']/a/img/@src")
                item["content_img"] = "https:" + item["content_img"][0] if len(item["content_img"]) > 0 else None
                item["author_img"] = div.xpath(".//div[@class='author clearfix']//img/@src")
                item["author_img"] = "https:" + item["author_img"][0] if len(item["author_img"]) > 0 else None
                item["stats_vote"] = div.xpath(".//span[@class='stats-vote']/i/text()")
                item["stats_vote"] = item["stats_vote"][0] if len(item["stats_vote"]) > 0 else None
                content_list.append(item)
            # return content_list
            self.content_queue.put(content_list)
            self.html_queue.task_done()

    def save_content_list(self):  # 保存数据
        while True:
            content_list = self.content_queue.get()
            with open("qiubai.txt", "a", encoding='utf-8') as f:
                for content in content_list:
                    # f.write(json.dumps(content, ensure_ascii=False))
                    f.write("\n".join(content["content"]) + "点赞数：" + content["stats_vote"])
                    f.write("\n")
            self.content_queue.task_done()

    def run(self):  # 实现主要逻辑
        thread_list = []
        # 1.构造url_list
        # url_list = self.get_url_list()
        t_geturl = threading.Thread(target=self.get_url_list)
        thread_list.append(t_geturl)
        # 2.遍历list，发送请求获取相应
        for i in range(7):  # 来7个线程来请求url地址
            t_parseurl = threading.Thread(target=self.parse_url)
            thread_list.append(t_parseurl)
        # for url in url_list:
        # html_str = self.parse_url(url)
        # 3.提取数据
        # content_list = self.get_content_list(html_str)
        for i in range(3):
            t_getcontent = threading.Thread(target=self.get_content_list)
            thread_list.append(t_getcontent)
        # 4.保存数据
        # self.save_content_list(content_list)
        t_save = threading.Thread(target=self.save_content_list)
        thread_list.append(t_save)
        for t in thread_list:
            t.setDaemon(True)  # 子线程设置为守护线程，表示该线程不重要，主线程结束则子线程也会结束
            t.start()
        for q in [self.url_queue, self.html_queue, self.content_queue]:
            q.join()  # 阻塞等待子线程结束
        print("主线程结束")


if __name__ == '__main__':
    if os.path.exists("qiubai.txt"):
        os.remove("qiubai.txt")
    qiubai = QuibaiSpdier()
    qiubai.run()
    

import requests

class TiebaSpider:
    def __init__(self, tieba_name):
        self.tieba_name = tieba_name
        self.url_temp = "https://tieba.baidu.com/f?kw=" + tieba_name + "&ie=utf-8&pn={}"
        self.headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36"}

    def get_url_list(self):  #构造url列表
        # url_list = []
        # for i in range(1000):
        #     url_list.append(self.url_temp.format(i*50))
        # return url_list
        return [self.url_temp.format(i*50) for i in range(1000)]

    def parse_url(self, url):  #发送请求获取相应
        print(url)
        response = requests.get(url, headers=self.headers)
        return response.content.decode()

    def save_html(self, html_str, page_num):
        file_path = "{}第{}页.html".format(self.tieba_name, page_num)
        with open(file_path, "w",encoding="utf-8") as f:
            f.write(html_str)

    def run(self):
        # 构造url列表
        url_list = self.get_url_list()
        # 遍历，发送请求，获取响应
        for url in url_list:
            html_str = self.parse_url(url)
            page_num = url_list.index(url)+1
            self.save_html(html_str, page_num)
        # 保存


if __name__ == '__main__':
    tieba_spider = TiebaSpider("李毅")
    tieba_spider.run()

proxies={
    "http":"http://xx.xx.xx.xx:xxx",
    "https":"https://xx.xx.xx.xx:xxxx",
}

session = requests.session()
response = session.get(url,headers)

# coding=utf-8
import requests
import json

class DoubanSpider:
    def __init__(self):
        self.headers = {
            "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1",
            "Referer": "https://m.douban.com/tv/american"}
        self.url_temp_list = [
            {
                "url_temp": "https://m.douban.com/rexxar/api/v2/subject_collection/tv_american/items?start{}&count=18&loc_id=108288",
                "type": "英美剧",
            },
            {
                "url_temp": "https://m.douban.com/rexxar/api/v2/subject_collection/tv_domestic/items?start{}&count=18&loc_id=108288",
                "type": "国产剧",
            },
            {
                "url_temp": "https://m.douban.com/rexxar/api/v2/subject_collection/tv_animation/items?start{}&count=18&loc_id=108288",
                "type": "动漫",
            },
            {
                "url_temp": "https://m.douban.com/rexxar/api/v2/subject_collection/tv_variety_show/items?start{}&count=18&loc_id=108288",
                "type": "综艺",
            }
        ]
        self.curIndex=1

    def parse_url(self, url):
        print(url)
        response = requests.get(url, headers=self.headers)
        return response.content.decode()

    def get_content_list(self, json_str):
        dict_ret = json.loads(json_str)
        content_list = dict_ret["subject_collection_items"]
        total = dict_ret["total"]
        return content_list, total

    def save_content_list(self, content_list):
        with open("douban.txt", "a", encoding='utf-8') as f:
            for content in content_list:
                #f.write(json.dumps(content, ensure_ascii=False))
                f.write(self.curIndex.__str__()+" :" + content["title"]+" 评分："+content["rating"]["value"].__str__())
                f.write("\n")
                self.curIndex += 1

    def run(self):  # 主逻辑
        for url_temp in self.url_temp_list:
            with open("douban.txt", "a", encoding='utf-8') as f:
                f.write(url_temp["type"] + ":================================================================")  # 分割线，方便数据查看
                f.write("\n")
            self.curIndex = 1
            num = 0
            total = 1  # 假设刚开始的条件成立
            while num < total + 18:
                # 1.构造一个start_rul
                url = url_temp["url_temp"].format(num)
                # 2.发送请求，获取响应
                json_str = self.parse_url(url)
                # 3.提取数据，保存
                content_list, total = self.get_content_list(json_str)
                self.save_content_list(content_list)
                # if len(content_list < 18):
                #    break
                # 4，构造下一页的url地址，进入2，3循环
                num += 18


if __name__ == '__main__':
    douban = DoubanSpider()
    douban.run()
    print("complete")

# coding=utf-8
import requests
from lxml import html


class QuibaiSpdier:
    def __init__(self):
        self.url_temp = "https://www.qiushibaike.com/hot/page/{}/"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36"}

    def get_url_list(self):
        return [self.url_temp.format(i) for i in range(1, 14)]

    def parse_url(self, url):
        response = requests.get(url, headers=self.headers)
        return response.content.decode()

    def get_content_list(self, html_str):
        html_elements = html.etree.HTML(html_str)
        div_list = html_elements.xpath("//div[@id='content-left']/div")
        content_list = []
        for div in div_list:
            item = {}
            item["content"] = div.xpath(".//div[@class='content']/span/text()")
            item["author_gender"] = div.xpath(".//div[contains(@class,'articleGender')]/@class")
            item["author_gender"] = item["author_gender"][0].split(" ")[-1].replace("Icon", "") if len(
                item["author_gender"]) > 0 else None
            item["content_img"] = div.xpath(".//div[@class='thumb']/a/img/@src")
            item["content_img"] = "https:" + item["content_img"][0] if len(item["content_img"]) > 0 else None
            item["author_img"] = div.xpath(".//div[@class='author clearfix']//img/@src")
            item["author_img"] = "https:" + item["author_img"][0] if len(item["author_img"]) > 0 else None
            item["stats_vote"] = div.xpath(".//span[@class='stats-vote']/i/text()")
            item["stats_vote"] = item["stats_vote"][0] if len(item["stats_vote"]) > 0 else None
            content_list.append(item)
        return content_list

    def save_content_list(self, content_list):
        with open("qiubai.txt", "a", encoding='utf-8') as f:
            for content in content_list:
                # f.write(json.dumps(content, ensure_ascii=False))
                f.write("\n".join(content["content"]) + "点赞数：" + content["stats_vote"])
                f.write("\n")

    def run(self):  # 实现主要逻辑
        # 1.构造url_list
        url_list = self.get_url_list()
        # 2.遍历list，发送请求获取相应
        for url in url_list:
            html_str = self.parse_url(url)
            # 3.提取数据
            content_list = self.get_content_list(html_str)
            # 4.保存数据
            self.save_content_list(content_list)


if __name__ == '__main__':
    qiubai = QuibaiSpdier()
    qiubai.run()