""" -*- coding: utf-8 -*- @Time :2022/04/12 17:10 @Author : Pengyou FU @blogs : https://blog.csdn.net/Echo_Code?spm=1000.2115.3001.5343 @github : https://github.com/FuSiry/OpenSA @WeChat : Fu_siry @License：Apache-2.0 license """ from sklearn.model_selection import train_test_split import numpy as np import pandas as pd #随机划分数据集 def random(data, label, test_ratio=0.2, random_state=123): """ :param data: shape (n_samples, n_features) :param label: shape (n_sample, ) :param test_size: the ratio of test_size, default: 0.2 :param random_state: the randomseed, default: 123 :return: X_train :(n_samples, n_features) X_test: (n_samples, n_features) y_train: (n_sample, ) y_test: (n_sample, ) """ X_train, X_test, y_train, y_test = train_test_split(data, label, test_size=test_ratio, random_state=random_state)#,stratify=label return X_train, X_test, y_train, y_test def spxy(data, label, test_size=0.2): """ :param data: shape (n_samples, n_features) :param label: shape (n_samples, ) :param test_size: the ratio of test_size, default: 0.2 :return: X_train :(n_samples, n_features) X_test: (n_samples, n_features) y_train: (n_samples, ) y_test: (n_samples, ) """ # 确保 data 和 label 是 NumPy 数组 data = data.to_numpy() if isinstance(data, pd.DataFrame) else data label = label.to_numpy() if isinstance(label, pd.Series) else label # 备份原始数据和标签 x_backup = data y_backup = label M = data.shape[0] N = round((1 - test_size) * M) samples = np.arange(M) # 归一化标签数据 label = (label - np.mean(label)) / np.std(label) D = np.zeros((M, M)) Dy = np.zeros((M, M)) # 计算样本之间的距离 for i in range(M - 1): xa = data[i, :] ya = label[i] for j in range((i + 1), M): xb = data[j, :] yb = label[j] D[i, j] = np.linalg.norm(xa - xb) Dy[i, j] = np.linalg.norm(ya - yb) # 距离归一化 Dmax = np.max(D) Dymax = np.max(Dy) D = D / Dmax + Dy / Dymax # 找到最远的两个点 maxD = D.max(axis=0) index_row = D.argmax(axis=0) index_column = maxD.argmax() m = np.zeros(N, dtype=int) m[0] = index_row[index_column] m[1] = index_column dminmax = np.zeros(N) dminmax[1] = D[m[0], m[1]] # 根据距离选择训练集 for i in range(2, N): pool = np.delete(samples, m[:i]) dmin = np.zeros(M - i) for j in range(M - i): indexa = pool[j] d = np.zeros(i) for k in range(i): indexb = m[k] if indexa < indexb: d[k] = D[indexa, indexb] else: d[k] = D[indexb, indexa] dmin[j] = np.min(d) dminmax[i] = np.max(dmin) index = np.argmax(dmin) m[i] = pool[index] m_complement = np.delete(samples, m) # 划分训练集和测试集 X_train = data[m, :] y_train = y_backup[m] X_test = data[m_complement, :] y_test = y_backup[m_complement] return X_train, X_test, y_train, y_test #利用kennard-stone算法划分数据集 def ks(data, label, test_size=0.2): """ :param data: shape (n_samples, n_features) :param label: shape (n_sample, ) :param test_size: the ratio of test_size, default: 0.2 :return: X_train: (n_samples, n_features) X_test: (n_samples, n_features) y_train: (n_samples, ) y_test: (n_samples, ) """ # 确保 data 和 label 是 NumPy 数组 data = data.to_numpy() if isinstance(data, pd.DataFrame) else data label = label.to_numpy() if isinstance(label, pd.Series) else label M = data.shape[0] N = round((1 - test_size) * M) samples = np.arange(M) D = np.zeros((M, M)) for i in range((M - 1)): xa = data[i, :] for j in range((i + 1), M): xb = data[j, :] D[i, j] = np.linalg.norm(xa - xb) maxD = np.max(D, axis=0) index_row = np.argmax(D, axis=0) index_column = np.argmax(maxD) m = np.zeros(N) m[0] = np.array(index_row[index_column]) m[1] = np.array(index_column) m = m.astype(int) dminmax = np.zeros(N) dminmax[1] = D[m[0], m[1]] for i in range(2, N): pool = np.delete(samples, m[:i]) dmin = np.zeros((M - i)) for j in range((M - i)): indexa = pool[j] d = np.zeros(i) for k in range(i): indexb = m[k] if indexa < indexb: d[k] = D[indexa, indexb] else: d[k] = D[indexb, indexa] dmin[j] = np.min(d) dminmax[i] = np.max(dmin) index = np.argmax(dmin) m[i] = pool[index] m_complement = np.delete(np.arange(data.shape[0]), m) X_train = data[m, :] y_train = label[m] X_test = data[m_complement, :] y_test = label[m_complement] return X_train, X_test, y_train, y_test # 分别使用一个回归、一个分类的公开数据集做为example def LoadNirtest(type): if type == "Rgs": CDataPath1 = r'G:\UAV\dazhou\20m\新,无条带\output.csv' data1 = np.loadtxt(open(CDataPath1, 'rb'), dtype=np.float64, delimiter=',', skiprows=0) data = data1[:, 2:] label = data1[:, 0] elif type == "Cls": path = r"G:\danzhu_test\rgb_refine\reflence\yellow_green_deepgreen\sum.csv" Nirdata = np.loadtxt(open(path, 'rb'), dtype=np.float64, delimiter=',', skiprows=0) data = Nirdata[1:, 1:463] label = Nirdata[1:,0] return data, label def SetSplit(method, data, label, test_size=0.3, randomseed=123): """ :param method: the method to split trainset and testset, include: random, kennard-stone(ks), spxy :param data: shape (n_samples, n_features) :param label: shape (n_sample, ) :param test_size: the ratio of test_size, default: 0.2 :return: X_train: (n_samples, n_features) X_test: (n_samples, n_features) y_train: (n_sample, ) y_test: (n_sample, ) """ if method == "random": X_train, X_test, y_train, y_test = random(data, label, test_size, randomseed) elif method == "spxy": X_train, X_test, y_train, y_test = spxy(data, label, test_size) elif method == "ks": X_train, X_test, y_train, y_test = ks(data, label, test_size) else: print("no this method of split dataset! ") return X_train, X_test, y_train, y_test