C# pythonnet(1)_传感器数据清洗算法

Python代码如下

import pandas as pd# 读取数据
data = pd.read_csv('data_row.csv')# 检查异常值
def detect_outliers(data):outliers = []for col in data.columns:q1 = data[col].quantile(0.25)q3 = data[col].quantile(0.75)iqr = q3 - q1lower_bound = q1 - 1.5 * iqrupper_bound = q3 + 1.5 * iqroutliers.extend(data[(data[col] < lower_bound) | (data[col] > upper_bound)].index)return list(set(outliers))outliers = detect_outliers(data)
print("异常数据数量：", len(outliers))
# 处理异常值
data.drop(outliers, inplace=True)# 保存清洗后的数据
data.to_csv('clean_data_row.csv', index=False)

下面我们修改成C#代码

创建控制台程序，Nuget安装 CsvHelper 和 pythonnet

public class Program
{const string PathToPythonDir = "D:\\Python311";const string DllOfPython = "python311.dll";static void Main(string[] args){// 数据清洗
        CleanData();}
/// <summary>/// 数据清洗/// </summary>static void CleanData(){var originDatas = ReadCsvWithCsvHelper("data_row.csv");var outliers = DetectOutliers(originDatas);var outlierHashset = new HashSet<int>(outliers);// 清洗过后的数据var cleanDatas = originDatas.Where((r, index) => !outlierHashset.Contains(index)).ToList();try{Runtime.PythonDLL = Path.Combine(PathToPythonDir, DllOfPython);PythonEngine.Initialize();using (Py.GIL()){dynamic pd = Py.Import("pandas");dynamic np = Py.Import("numpy");dynamic plt = Py.Import("matplotlib.pyplot");dynamic fft = Py.Import("scipy.fftpack");dynamic oData = np.array(originDatas.ToArray());int oDataLength = oData.__len__();dynamic data = np.array(cleanDatas.ToArray());int dataLength = data.__len__();// 绘制原始数据图和清洗后数据图plt.figure(figsize: new dynamic[] { 12, 6 });// 原始数据图plt.subplot(1, 2, 1);plt.plot(np.arange(oDataLength), oData);plt.title("Original Datas");// 清洗后数据图plt.subplot(1, 2, 2);plt.plot(np.arange(dataLength), data);plt.title("Clean Datas");// 布局调整，防止重叠
                plt.tight_layout();// 显示图表
                plt.show();}}catch (Exception e){Console.WriteLine("报错了：" + e.Message + "\r\n" + e.StackTrace);}}/// <summary>/// 检测异常值/// </summary>/// <param name="datas">原始数据集合</param>/// <returns>返回异常值在集合中的索引</returns>static List<int> DetectOutliers(List<double[]> datas){List<int> outliers = new List<int>();var first = datas.First();for (int i = 0; i < first.Length; i++){var values = datas.AsEnumerable().Select((row, index) => Tuple.Create(row[i], index)).ToArray();double q1 = Enumerable.OrderBy(values, x => x.Item1).ElementAt((int)(values.Length * 0.25)).Item1;double q3 = Enumerable.OrderBy(values, x => x.Item1).ElementAt((int)(values.Length * 0.75)).Item1;double iqr = q3 - q1;double lowerBound = q1 - 1.5 * iqr;double upperBound = q3 + 1.5 * iqr;outliers.AddRange(values.AsEnumerable().Where(row => row.Item1 < lowerBound || row.Item1 > upperBound).Select(row => row.Item2));}return outliers.Distinct().ToList();}/// <summary>/// 读取CSV数据/// </summary>/// <param name="filePath">文件路径</param>/// <returns>文件中数据集合，都是double类型</returns>static List<double[]> ReadCsvWithCsvHelper(string filePath){using (var reader = new StreamReader(filePath))using (var csv = new CsvReader(reader, CultureInfo.InvariantCulture)){var result = new List<double[]>();// 如果你的CSV文件有标题行，可以调用ReadHeader来读取它们
            csv.Read();csv.ReadHeader();while (csv.Read()){result.Add(new double[] {csv.GetField<double>(0),csv.GetField<double>(1),csv.GetField<double>(2),});}return result;}}
}

以下是运行后结果，左边是原始数据折线图，右边是清洗后数据折线图

 源代码：https://gitee.com/Karl_Albright/csharp-demo/tree/master/PythonnetDemo/PythonnetClearData

 

 

 

抽稀算法

def down_sampling(sig,factor=2, axis=0):'''降采样Inputs:sig --- numpy array, 信号数据数组factor --- int, 降采样倍率axis --- int, 沿着哪个轴进行降采样'''Temp=[':']*sig.ndimTemp[axis]='::'+str(factor)return eval('sig['+','.join(Temp)+']')

/// <summary>
/// 降采样，其实就是抽稀算法
/// </summary>
static List<double[]> DownSampling(int factor = 2, int axis = 0)
{if (axis != 0 && axis != 1)throw new ArgumentException("Axis must be 0 or 1 for a 2D array.");var datas = ReadCsvWithCsvHelper("clean_data_row3.csv");int dim0 = datas.Count;var first = datas.First();int dim1 = first.Length;var result = new List<double[]>();if (axis == 0){var xAxis = dim0 / factor;var yAxis = dim1;for (int i = 0; i < xAxis; i++){result.Add(datas[i * factor]);}}else if (axis == 1){var xAxis = dim0;var yAxis = dim1 / factor;var item = new double[yAxis];for (int i = 0; i < xAxis; i++){var deviceData = datas[i];for (int j = 0; j < yAxis; j++){item[j] = deviceData[j * factor];}result.Add(item);}}return result;
}

 源代码：https://gitee.com/Karl_Albright/csharp-demo/tree/master/PythonnetDemo/PythonnetClearData