{"id":89275,"date":"2026-08-02T19:17:07","date_gmt":"2026-08-02T11:17:07","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/89275.html"},"modified":"2026-08-02T19:17:07","modified_gmt":"2026-08-02T11:17:07","slug":"%e7%ac%ac14%e8%af%be%ef%bc%9apyspark%ef%bd%9c%e5%86%85%e7%bd%ae%e5%87%bd%e6%95%b0%e5%85%a8%e8%a7%a3%e3%80%90%e5%ad%97%e7%ac%a6%e4%b8%b2%e6%97%a5%e6%9c%9f%e8%81%9a%e5%90%88%e6%9d%a1%e4%bb%b6%e7%aa%97","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/89275.html","title":{"rendered":"\u7b2c14\u8bfe\uff1aPySpark\uff5c\u5185\u7f6e\u51fd\u6570\u5168\u89e3\u3010\u5b57\u7b26\u4e32\u65e5\u671f\u805a\u5408\u6761\u4ef6\u7a97\u53e3\u51fd\u6570\u5b9e\u6218\u3011"},"content":{"rendered":"<p><img decoding=\"async\" src=\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/08\/20260802111705-6a6f273153ea9.jpg\" alt=\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\" \/><\/p>\n<\/p>\n<h4>\u6587\u7ae0\u76ee\u5f55<\/h4>\n<ul>\n<li>\n<ul>\n<li>\u4e00\u3001\u8bfe\u524d\u5bfc\u8bfb<\/li>\n<li>\u4e8c\u3001\u5b66\u4e60\u76ee\u6807<\/li>\n<li>\u4e09\u3001\u6838\u5fc3\u7406\u8bba\u77e5\u8bc6\u70b9<\/li>\n<li>\u56db\u3001\u539f\u7406\u901a\u4fd7\u8bb2\u89e3<\/li>\n<li>\n<ul>\n<li>4.1 \u5185\u7f6e\u51fd\u6570\u4e3a\u4f55\u6bd4UDF\u5feb\u5f97\u591a&#xff1f;<\/li>\n<li>4.2 \u51fd\u6570\u7ec4\u5408\u4e0e\u94fe\u5f0f\u8c03\u7528<\/li>\n<li>4.3 \u7a97\u53e3\u51fd\u6570\u7684\u7269\u7406\u6267\u884c<\/li>\n<\/ul>\n<\/li>\n<li>\u4e94\u3001\u91cd\u70b9\u6982\u5ff5\u62c6\u89e3<\/li>\n<li>\n<ul>\n<li>5.1 \u5b57\u7b26\u4e32\u51fd\u6570\u8be6\u89e3<\/li>\n<li>5.2 \u65e5\u671f\u65f6\u95f4\u51fd\u6570\u8be6\u89e3<\/li>\n<li>5.3 \u805a\u5408\u51fd\u6570\u8be6\u89e3<\/li>\n<li>5.4 \u6761\u4ef6\u51fd\u6570\u8be6\u89e3<\/li>\n<li>5.5 \u7a97\u53e3\u51fd\u6570\u8be6\u89e3<\/li>\n<li>5.6 \u96c6\u5408\u51fd\u6570&#xff08;\u6570\u7ec4\u4e0eMap&#xff09;<\/li>\n<\/ul>\n<\/li>\n<li>\u516d\u3001\u6613\u9519\u70b9\u907f\u5751<\/li>\n<li>\n<ul>\n<li>6.1 \u6df7\u6dc6&#096;substring&#096;\u7684\u7d22\u5f15\u8d77\u59cb<\/li>\n<li>6.2 \u65e5\u671f\u683c\u5f0f\u5b57\u7b26\u4e32\u5927\u5c0f\u5199\u654f\u611f<\/li>\n<li>6.3 \u7a97\u53e3\u51fd\u6570\u4e2d&#096;order by&#096;\u7f3a\u5c11\u4f1a\u5bfc\u81f4\u6574\u4e2a\u5206\u533a\u4f5c\u4e3a\u4e00\u4e2a\u7a97\u53e3<\/li>\n<li>6.4 &#096;collect_list&#096;\u7ed3\u679c\u65e0\u5e8f\u4e14\u53ef\u80fd\u53d7\u5206\u533a\u5f71\u54cd<\/li>\n<li>6.5 \u5728\u805a\u5408\u524d\u4f7f\u7528&#096;explode&#096;\u5bfc\u81f4\u6570\u636e\u81a8\u80c0<\/li>\n<li>6.6 \u4f7f\u7528&#096;when&#096;\u65f6\u9057\u6f0f&#096;otherwise&#096;<\/li>\n<li>6.7 &#096;coalesce&#096;\u4e0e&#096;repartition&#096;\u6df7\u6dc6<\/li>\n<\/ul>\n<\/li>\n<li>\u4e03\u3001\u5b8c\u6574\u5b9e\u6218\u6848\u4f8b<\/li>\n<li>\u516b\u3001\u4ee3\u7801\u9010\u884c\u89e3\u6790<\/li>\n<li>\n<ul>\n<li>8.1 \u6b63\u5219\u63d0\u53d6\u5b57\u6bb5<\/li>\n<li>8.2 \u5b57\u7b26\u4e32\u5206\u5272\u4e0e\u53d6\u5143\u7d20<\/li>\n<li>8.3 \u65e5\u671f\u8f6c\u6362<\/li>\n<li>8.4 \u6761\u4ef6\u5206\u7c7b<\/li>\n<li>8.5 \u8fd1\u4f3c\u53bb\u91cd\u8ba1\u6570<\/li>\n<li>8.6 \u900f\u89c6&#xff08;pivot&#xff09;<\/li>\n<li>8.7 \u7a97\u53e3\u51fd\u6570<\/li>\n<li>8.8 \u6570\u7ec4\u5c55\u5f00<\/li>\n<li>8.9 UDF vs \u5185\u7f6e\u51fd\u6570\u6027\u80fd\u5bf9\u6bd4<\/li>\n<\/ul>\n<\/li>\n<li>\u4e5d\u3001\u4e1a\u52a1\u573a\u666f\u843d\u5730\u5e94\u7528<\/li>\n<li>\n<ul>\n<li>9.1 \u573a\u666f\u4e00&#xff1a;\u7528\u6237\u6ce8\u518c\u65e5\u5fd7\u4e2d\u7684\u8bbe\u5907\u4fe1\u606f\u89e3\u6790<\/li>\n<li>9.2 \u573a\u666f\u4e8c&#xff1a;\u7535\u5546\u8ba2\u5355\u7684RFM\u5206\u5c42<\/li>\n<li>9.3 \u573a\u666f\u4e09&#xff1a;\u5b9e\u65f6\u6d41\u5904\u7406\u4e2d\u7684JSON\u89e3\u6790<\/li>\n<li>9.4 \u573a\u666f\u56db&#xff1a;\u65e5\u5fd7\u8def\u5f84\u5206\u6790\u4e2d\u7684\u8def\u5f84\u62c6\u5206<\/li>\n<\/ul>\n<\/li>\n<li>\u5341\u3001\u5e38\u89c1\u62a5\u9519\u6392\u67e5<\/li>\n<li>\n<ul>\n<li>10.1 &#096;to_timestamp&#096;\u8fd4\u56denull<\/li>\n<li>10.2 \u7a97\u53e3\u51fd\u6570\u62a5\u9519&#096;Window function without frame&#096;<\/li>\n<li>10.3 &#096;regexp_extract&#096;\u63d0\u53d6\u4e0d\u5230\u503c\u8fd4\u56de\u7a7a\u5b57\u7b26\u4e32<\/li>\n<li>10.4 \u4f7f\u7528&#096;collect_list&#096;\u5bfc\u81f4\u5185\u5b58\u6ea2\u51fa<\/li>\n<li>10.5 &#096;explode&#096;\u540e\u6570\u636e\u81a8\u80c0\u5bfc\u81f4Shuffle\u6570\u636e\u91cf\u5de8\u5927<\/li>\n<\/ul>\n<\/li>\n<li>\u5341\u4e00\u3001\u672c\u8282\u8bfe\u77e5\u8bc6\u70b9\u603b\u7ed3<\/li>\n<li>\n<ul>\n<li>\u51fd\u6570\u5206\u7c7b\u901f\u67e5\u8868<\/li>\n<li>\u6027\u80fd\u4f18\u5316\u8981\u70b9<\/li>\n<\/ul>\n<\/li>\n<li>\u5341\u4e8c\u3001\u8bfe\u540e\u601d\u8003\u4f5c\u4e1a<\/li>\n<li>\n<ul>\n<li>\u4f5c\u4e1a\u4e00&#xff1a;\u7406\u8bba\u7406\u89e3\u9898<\/li>\n<li>\u4f5c\u4e1a\u4e8c&#xff1a;\u4ee3\u7801\u5b9e\u8df5\u9898<\/li>\n<li>\u4f5c\u4e1a\u4e09&#xff1a;\u573a\u666f\u5e94\u7528\u9898<\/li>\n<li>\u4f5c\u4e1a\u56db&#xff1a;\u62d3\u5c55\u7814\u7a76<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<\/li>\n<li>&#x1f517;\u300a20\u8282\u8bfe PySpark \u4ece\u5165\u95e8\u5230\u7cbe\u901a\u300b\u7cfb\u5217\u8bfe\u7a0b\u5bfc\u822a<\/li>\n<\/ul>\n<hr \/>\n<h3>\u4e00\u3001\u8bfe\u524d\u5bfc\u8bfb<\/h3>\n<p>\u5728\u524d\u9762\u7684\u8bfe\u7a0b\u4e2d&#xff0c;\u6211\u4eec\u5df2\u7ecf\u5b66\u4e60\u4e86DataFrame\u7684\u57fa\u7840\u64cd\u4f5c\u548cSpark SQL\u7684\u5b8c\u6574\u8bed\u6cd5\u3002\u4f60\u53ef\u80fd\u5df2\u7ecf\u53d1\u73b0&#xff0c;\u5728\u7f16\u5199DataFrame\u4ee3\u7801\u65f6&#xff0c;\u5927\u91cf\u7684\u6570\u636e\u5904\u7406\u903b\u8f91\u90fd\u4f9d\u8d56\u4e8epyspark.sql.functions\u6a21\u5757\u4e2d\u7684\u5185\u7f6e\u51fd\u6570\u3002\u4f8b\u5982&#xff0c;\u7528when\u5b9e\u73b0\u6761\u4ef6\u5224\u65ad&#xff0c;\u7528date_add\u5904\u7406\u65e5\u671f&#xff0c;\u7528row_number\u8fdb\u884c\u6392\u540d\u3002\u8fd9\u4e9b\u5185\u7f6e\u51fd\u6570\u4e0d\u4ec5\u8ba9\u4ee3\u7801\u66f4\u7b80\u6d01&#xff0c;\u66f4\u91cd\u8981\u7684\u662f\u2014\u2014\u5b83\u4eec\u7684\u6267\u884c\u6548\u7387\u8fdc\u8d85\u81ea\u5b9a\u4e49UDF&#xff0c;\u56e0\u4e3a\u5185\u7f6e\u51fd\u6570\u662f\u76f4\u63a5\u5728JVM\u4e2d\u6267\u884c\u7684&#xff0c;\u4e0d\u9700\u8981Python\u548cJVM\u4e4b\u95f4\u7684\u5e8f\u5217\u5316\u5f00\u9500\u3002<\/p>\n<p>\u7136\u800c&#xff0c;\u5185\u7f6e\u51fd\u6570\u7684\u6570\u91cf\u6781\u5176\u5e9e\u5927&#xff08;\u8d85\u8fc7200\u4e2a&#xff09;&#xff0c;\u5f88\u591a\u521d\u5b66\u8005\u53ea\u4f1a\u7528\u5176\u4e2d\u51e0\u4e2a&#xff08;\u5982col\u3001sum\u3001avg&#xff09;&#xff0c;\u9047\u5230\u7a0d\u5fae\u590d\u6742\u7684\u9700\u6c42\u5c31\u675f\u624b\u65e0\u7b56&#xff0c;\u751a\u81f3\u53bb\u5199\u4f4e\u6548\u7684UDF\u3002\u4f8b\u5982&#xff0c;\u60f3\u8981\u63d0\u53d6\u5b57\u7b26\u4e32\u4e2d\u7684\u65e5\u671f\u3001\u8ba1\u7b97\u4e24\u4e2a\u65e5\u671f\u7684\u5dee\u503c\u3001\u5b9e\u73b0\u884c\u8f6c\u5217&#xff08;pivot&#xff09;\u3001\u8ba1\u7b97\u7d2f\u8ba1\u548c\u7b49&#xff0c;\u5b9e\u9645\u4e0a\u90fd\u6709\u5bf9\u5e94\u7684\u5185\u7f6e\u51fd\u6570\u6216\u7ec4\u5408\u7528\u6cd5\u3002<\/p>\n<p>\u672c\u8282\u8bfe\u5c06\u7cfb\u7edf\u6027\u5730\u8bb2\u89e3PySpark\u4e2d\u6700\u5e38\u7528\u7684\u5185\u7f6e\u51fd\u6570&#xff0c;\u6309\u7167\u529f\u80fd\u6a21\u5757\u5206\u4e3a&#xff1a;\u5b57\u7b26\u4e32\u51fd\u6570\u3001\u65e5\u671f\u65f6\u95f4\u51fd\u6570\u3001\u805a\u5408\u51fd\u6570\u3001\u6761\u4ef6\u51fd\u6570\u3001\u7a97\u53e3\u51fd\u6570\u3001\u96c6\u5408\u51fd\u6570\u7b49\u3002\u6bcf\u4e2a\u51fd\u6570\u90fd\u4f1a\u914d\u5408\u5b9e\u6218\u4ee3\u7801\u548c\u4e1a\u52a1\u573a\u666f&#xff0c;\u5e2e\u52a9\u4f60\u5f7b\u5e95\u638c\u63e1\u5e76\u80fd\u5728\u5b9e\u9645\u5de5\u4f5c\u4e2d\u7075\u6d3b\u8fd0\u7528\u3002\u5b66\u5b8c\u8fd9\u8282\u8bfe&#xff0c;\u4f60\u5c06\u4e0d\u518d\u9700\u8981\u4e3a\u7b80\u5355\u7684\u6570\u636e\u8f6c\u6362\u7f16\u5199UDF&#xff0c;\u4ee3\u7801\u6027\u80fd\u5c06\u63d0\u5347\u4e00\u4e2a\u6570\u91cf\u7ea7\u3002<\/p>\n<h3>\u4e8c\u3001\u5b66\u4e60\u76ee\u6807<\/h3>\n<p>\u5b8c\u6210\u672c\u8282\u8bfe\u7684\u5b66\u4e60\u540e&#xff0c;\u4f60\u5c06\u80fd\u591f&#xff1a;<\/p>\n<li>\u638c\u63e1\u6838\u5fc3\u5b57\u7b26\u4e32\u51fd\u6570&#xff1a;concat\u3001split\u3001regexp_extract\u3001substring\u3001trim\u3001upper\/lower\u3001length\u3001replace\u7b49<\/li>\n<li>\u638c\u63e1\u6838\u5fc3\u65e5\u671f\u51fd\u6570&#xff1a;to_date\/to_timestamp\u3001date_format\u3001date_add\/date_sub\u3001datediff\u3001year\/month\/day\u3001current_date\u3001unix_timestamp\u7b49<\/li>\n<li>\u638c\u63e1\u9ad8\u7ea7\u805a\u5408\u51fd\u6570&#xff1a;collect_list\u3001collect_set\u3001first\/last\u3001grouping\u3001pivot\u7b49<\/li>\n<li>\u638c\u63e1\u6761\u4ef6\u51fd\u6570&#xff1a;when\u3001otherwise\u3001coalesce\u3001ifnull\u3001nanvl\u7b49<\/li>\n<li>\u638c\u63e1\u7a97\u53e3\u51fd\u6570&#xff1a;row_number\u3001rank\u3001dense_rank\u3001lag\/lead\u3001ntile\u3001\u805a\u5408\u7a97\u53e3\u51fd\u6570&#xff08;sum over\u7b49&#xff09;<\/li>\n<li>\u638c\u63e1\u96c6\u5408\u51fd\u6570&#xff1a;array_contains\u3001size\u3001explode\u3001posexplode\u3001array_distinct\u7b49\u5904\u7406\u6570\u7ec4\u548cMap<\/li>\n<li>\u7406\u89e3\u5185\u7f6e\u51fd\u6570\u4f18\u4e8eUDF\u7684\u539f\u7406&#xff1a;\u907f\u514d\u5e8f\u5217\u5316\u5f00\u9500&#xff0c;\u5229\u7528Catalyst\u4f18\u5316\u548c\u4ee3\u7801\u751f\u6210<\/li>\n<li>\u80fd\u591f\u5c06\u590d\u6742\u4e1a\u52a1\u903b\u8f91\u7528\u5185\u7f6e\u51fd\u6570\u94fe\u5f0f\u8868\u8fbe&#xff0c;\u5199\u51fa\u9ad8\u6548\u53ef\u8bfb\u7684\u4ee3\u7801<\/li>\n<h3>\u4e09\u3001\u6838\u5fc3\u7406\u8bba\u77e5\u8bc6\u70b9<\/h3>\n<table>\n<tr>\u51fd\u6570\u7c7b\u522b\u5e38\u7528\u51fd\u6570\u8bf4\u660e<\/tr>\n<tbody>\n<tr>\n<td>\u5b57\u7b26\u4e32<\/td>\n<td>concat, concat_ws, length, lower\/upper, ltrim\/rtrim\/trim, reverse, substring, split, regexp_extract, regexp_replace, replace, instr, locate<\/td>\n<td>\u5b57\u7b26\u4e32\u62fc\u63a5\u3001\u622a\u53d6\u3001\u6b63\u5219\u5339\u914d\u66ff\u6362<\/td>\n<\/tr>\n<tr>\n<td>\u65e5\u671f\u65f6\u95f4<\/td>\n<td>current_date, current_timestamp, to_date, to_timestamp, date_format, year, month, dayofmonth, dayofweek, weekofyear, datediff, months_between, date_add, date_sub, add_months, last_day, trunc, from_unixtime, unix_timestamp<\/td>\n<td>\u65e5\u671f\u89e3\u6790\u3001\u683c\u5f0f\u5316\u3001\u8ba1\u7b97\u5dee\u503c<\/td>\n<\/tr>\n<tr>\n<td>\u805a\u5408<\/td>\n<td>sum, avg, count, max, min, stddev, variance, collect_list, collect_set, first, last, approx_count_distinct<\/td>\n<td>\u5e38\u89c4\u805a\u5408\u53ca\u9ad8\u7ea7\u805a\u5408<\/td>\n<\/tr>\n<tr>\n<td>\u6761\u4ef6<\/td>\n<td>when, otherwise, coalesce, isnull, isnan, if, ifnull, nullif, nanvl<\/td>\n<td>\u6761\u4ef6\u5224\u65ad\u3001\u7a7a\u503c\u5904\u7406<\/td>\n<\/tr>\n<tr>\n<td>\u7a97\u53e3<\/td>\n<td>row_number, rank, dense_rank, percent_rank, ntile, cume_dist, lag, lead, \u805a\u5408\u51fd\u6570\u914d\u5408over<\/td>\n<td>\u6392\u540d\u3001\u504f\u79fb\u3001\u7d2f\u8ba1\u8ba1\u7b97<\/td>\n<\/tr>\n<tr>\n<td>\u96c6\u5408(\u6570\u7ec4\/Map)<\/td>\n<td>array, array_contains, size, explode, posexplode, array_distinct, array_remove, array_union, array_intersect, array_join, map_keys, map_values<\/td>\n<td>\u5904\u7406\u590d\u6742\u6570\u636e\u7c7b\u578b<\/td>\n<\/tr>\n<tr>\n<td>\u6570\u5b66<\/td>\n<td>abs, ceil, floor, round, pow, sqrt, rand<\/td>\n<td>\u6570\u503c\u8ba1\u7b97<\/td>\n<\/tr>\n<tr>\n<td>\u5176\u4ed6<\/td>\n<td>col, lit, expr, struct, to_json, from_json<\/td>\n<td>\u8f85\u52a9\u51fd\u6570<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h3>\u56db\u3001\u539f\u7406\u901a\u4fd7\u8bb2\u89e3<\/h3>\n<h4>4.1 \u5185\u7f6e\u51fd\u6570\u4e3a\u4f55\u6bd4UDF\u5feb\u5f97\u591a&#xff1f;<\/h4>\n<p>\u5f53\u4f60\u4f7f\u7528Python UDF\u65f6&#xff0c;\u4f8b\u5982&#xff1a;<\/p>\n<p><span class=\"token decorator annotation punctuation\">&#064;udf<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;string&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">def<\/span> <span class=\"token function\">my_upper<\/span><span class=\"token punctuation\">(<\/span>s<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token keyword\">return<\/span> s<span class=\"token punctuation\">.<\/span>upper<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>Spark\u5728\u6267\u884c\u65f6\u9700\u8981&#xff1a;<\/p>\n<li>\u5c06DataFrame\u4e2d\u7684\u6bcf\u4e00\u884c&#xff08;JVM\u4e2d\u7684Row\u5bf9\u8c61&#xff09;\u5e8f\u5217\u5316\u4e3aPython\u5bf9\u8c61&#xff08;\u53cd\u5e8f\u5217\u5316&#xff09;<\/li>\n<li>\u5c06\u6570\u636e\u4f20\u9012\u7ed9Python\u8fdb\u7a0b<\/li>\n<li>Python\u51fd\u6570\u6267\u884c\u8ba1\u7b97<\/li>\n<li>\u5c06\u7ed3\u679c\u5e8f\u5217\u5316\u56deJVM<\/li>\n<li>JVM\u8fdb\u4e00\u6b65\u5904\u7406<\/li>\n<p>\u8fd9\u4e2a\u8fc7\u7a0b\u6d89\u53ca\u591a\u6b21\u5e8f\u5217\u5316\u548c\u8de8\u8fdb\u7a0b\u901a\u4fe1&#xff0c;\u5f00\u9500\u6781\u5927\u3002<\/p>\n<p>\u800c\u5185\u7f6e\u51fd\u6570&#xff08;\u5982upper(col(&#034;name&#034;))&#xff09;\u5b8c\u5168\u5728JVM\u5185\u90e8\u6267\u884c&#xff0c;\u76f4\u63a5\u64cd\u4f5c\u4e8c\u8fdb\u5236\u6570\u636e&#xff08;Tungsten\u683c\u5f0f&#xff09;&#xff0c;\u65e0\u9700\u5e8f\u5217\u5316&#xff0c;\u5e76\u4e14\u53ef\u4ee5\u53c2\u4e0eCatalyst\u4f18\u5316\u5668\u7684\u8868\u8fbe\u5f0f\u91cd\u5199\u548c\u4ee3\u7801\u751f\u6210&#xff0c;\u6700\u7ec8\u751f\u6210\u9ad8\u5ea6\u4f18\u5316\u7684Java\u5b57\u8282\u7801\u3002<\/p>\n<p>\u56e0\u6b64&#xff0c;\u80fd\u7528\u5185\u7f6e\u51fd\u6570\u89e3\u51b3\u7684\u573a\u666f&#xff0c;\u7edd\u5bf9\u4e0d\u8981\u5199UDF\u3002<\/p>\n<h4>4.2 \u51fd\u6570\u7ec4\u5408\u4e0e\u94fe\u5f0f\u8c03\u7528<\/h4>\n<p>\u5185\u7f6e\u51fd\u6570\u901a\u5e38\u662f\u8868\u8fbe\u5f0f&#xff08;Column\u7c7b\u578b&#xff09;&#xff0c;\u53ef\u4ee5\u8fdb\u884c\u7ec4\u5408\u3002\u4f8b\u5982&#xff1a;<\/p>\n<p>df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;domain&#034;<\/span><span class=\"token punctuation\">,<\/span> split<span class=\"token punctuation\">(<\/span>regexp_extract<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;email&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;&#064;(.*)&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\\\\\\\\.&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>getItem<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>\u8fd9\u79cd\u94fe\u5f0f\u8c03\u7528\u5728\u903b\u8f91\u4e0a\u6784\u5efa\u4e86\u4e00\u68f5\u8868\u8fbe\u5f0f\u6811&#xff0c;Catalyst\u4f18\u5316\u5668\u80fd\u591f\u5bf9\u8fd9\u68f5\u6811\u8fdb\u884c\u5e38\u91cf\u6298\u53e0\u3001\u8c13\u8bcd\u4e0b\u63a8\u7b49\u4f18\u5316\u3002<\/p>\n<h4>4.3 \u7a97\u53e3\u51fd\u6570\u7684\u7269\u7406\u6267\u884c<\/h4>\n<p>\u7a97\u53e3\u51fd\u6570\u9700\u8981\u5c06\u6570\u636e\u6309\u7167PARTITION BY\u5206\u7ec4&#xff0c;\u5728\u6bcf\u4e2a\u5206\u7ec4\u5185\u6309\u7167ORDER BY\u6392\u5e8f&#xff0c;\u7136\u540e\u8ba1\u7b97\u6392\u540d\u6216\u805a\u5408\u3002Spark\u4f1a\u901a\u8fc7WindowExec\u7269\u7406\u7b97\u5b50\u6765\u5b8c\u6210&#xff0c;\u5b83\u4f1a\u5bf9\u6570\u636e\u8fdb\u884c\u6392\u5e8f&#xff08;\u53ef\u80fd\u89e6\u53d1Shuffle&#xff09;&#xff0c;\u7136\u540e\u5728\u6bcf\u4e2a\u5206\u533a\u5185\u9010\u884c\u8ba1\u7b97\u3002\u5408\u7406\u4f7f\u7528\u7a97\u53e3\u51fd\u6570\u53ef\u4ee5\u907f\u514d\u590d\u6742\u7684\u81ea\u8fde\u63a5\u3002<\/p>\n<h3>\u4e94\u3001\u91cd\u70b9\u6982\u5ff5\u62c6\u89e3<\/h3>\n<h4>5.1 \u5b57\u7b26\u4e32\u51fd\u6570\u8be6\u89e3<\/h4>\n<table>\n<tr>\u51fd\u6570\u793a\u4f8b\u8f93\u51fa<\/tr>\n<tbody>\n<tr>\n<td>concat<\/td>\n<td>concat(col(&#034;first&#034;), lit(&#034; &#034;), col(&#034;last&#034;))<\/td>\n<td>\u201cJohn Doe\u201d<\/td>\n<\/tr>\n<tr>\n<td>concat_ws<\/td>\n<td>concat_ws(&#034;-&#034;, col(&#034;year&#034;), col(&#034;month&#034;))<\/td>\n<td>\u201c2024-01\u201d<\/td>\n<\/tr>\n<tr>\n<td>length<\/td>\n<td>length(col(&#034;name&#034;))<\/td>\n<td>5<\/td>\n<\/tr>\n<tr>\n<td>lower\/upper<\/td>\n<td>upper(col(&#034;name&#034;))<\/td>\n<td>\u201cALICE\u201d<\/td>\n<\/tr>\n<tr>\n<td>trim<\/td>\n<td>trim(col(&#034;name&#034;))<\/td>\n<td>\u53bb\u9664\u9996\u5c3e\u7a7a\u683c<\/td>\n<\/tr>\n<tr>\n<td>substring<\/td>\n<td>substring(col(&#034;date&#034;), 1, 10)<\/td>\n<td>\u622a\u53d6\u524d10\u4e2a\u5b57\u7b26<\/td>\n<\/tr>\n<tr>\n<td>split<\/td>\n<td>split(col(&#034;email&#034;), &#034;&#064;&#034;)<\/td>\n<td>\u8fd4\u56de\u6570\u7ec4<\/td>\n<\/tr>\n<tr>\n<td>regexp_extract<\/td>\n<td>regexp_extract(col(&#034;url&#034;), &#034;id&#061;(\\\\\\\\d&#043;)&#034;, 1)<\/td>\n<td>\u63d0\u53d6\u6570\u5b57<\/td>\n<\/tr>\n<tr>\n<td>regexp_replace<\/td>\n<td>regexp_replace(col(&#034;text&#034;), &#034;\\\\\\\\s&#043;&#034;, &#034; &#034;)<\/td>\n<td>\u66ff\u6362\u7a7a\u767d<\/td>\n<\/tr>\n<tr>\n<td>replace<\/td>\n<td>replace(col(&#034;phone&#034;), &#034;-&#034;, &#034;&#034;)<\/td>\n<td>\u5220\u9664\u8fde\u5b57\u7b26<\/td>\n<\/tr>\n<tr>\n<td>instr<\/td>\n<td>instr(col(&#034;email&#034;), &#034;&#064;&#034;)<\/td>\n<td>\u8fd4\u56de&#064;\u4f4d\u7f6e\u7d22\u5f15<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u6ce8\u610f&#xff1a;\u5b57\u7b26\u4e32\u7d22\u5f15\u4ece1\u5f00\u59cb&#xff08;\u4e0eSQL\u4e00\u81f4&#xff0c;\u4e0ePython\u4e0d\u540c&#xff09;\u3002<\/p>\n<h4>5.2 \u65e5\u671f\u65f6\u95f4\u51fd\u6570\u8be6\u89e3<\/h4>\n<table>\n<tr>\u51fd\u6570\u793a\u4f8b\u8bf4\u660e<\/tr>\n<tbody>\n<tr>\n<td>current_date()<\/td>\n<td>\u65e0\u53c2\u6570<\/td>\n<td>\u8fd4\u56de\u5f53\u524d\u65e5\u671f<\/td>\n<\/tr>\n<tr>\n<td>current_timestamp()<\/td>\n<td><\/td>\n<td>\u8fd4\u56de\u5f53\u524d\u65f6\u95f4\u6233<\/td>\n<\/tr>\n<tr>\n<td>to_date<\/td>\n<td>to_date(col(&#034;date_str&#034;), &#034;yyyy-MM-dd&#034;)<\/td>\n<td>\u5b57\u7b26\u4e32\u8f6c\u65e5\u671f<\/td>\n<\/tr>\n<tr>\n<td>to_timestamp<\/td>\n<td>to_timestamp(col(&#034;ts_str&#034;), &#034;yyyy-MM-dd HH:mm:ss&#034;)<\/td>\n<td>\u5b57\u7b26\u4e32\u8f6c\u65f6\u95f4\u6233<\/td>\n<\/tr>\n<tr>\n<td>date_format<\/td>\n<td>date_format(col(&#034;date&#034;), &#034;yyyyMM&#034;)<\/td>\n<td>\u683c\u5f0f\u5316\u4e3a\u5b57\u7b26\u4e32<\/td>\n<\/tr>\n<tr>\n<td>year\/month\/day<\/td>\n<td>year(col(&#034;date&#034;))<\/td>\n<td>\u63d0\u53d6\u90e8\u5206<\/td>\n<\/tr>\n<tr>\n<td>datediff<\/td>\n<td>datediff(col(&#034;end&#034;), col(&#034;start&#034;))<\/td>\n<td>\u65e5\u671f\u5dee&#xff08;\u5929&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>months_between<\/td>\n<td>months_between(col(&#034;end&#034;), col(&#034;start&#034;))<\/td>\n<td>\u6708\u4efd\u5dee<\/td>\n<\/tr>\n<tr>\n<td>date_add<\/td>\n<td>date_add(col(&#034;date&#034;), 7)<\/td>\n<td>\u52a07\u5929<\/td>\n<\/tr>\n<tr>\n<td>date_sub<\/td>\n<td>date_sub(col(&#034;date&#034;), 7)<\/td>\n<td>\u51cf7\u5929<\/td>\n<\/tr>\n<tr>\n<td>add_months<\/td>\n<td>add_months(col(&#034;date&#034;), 3)<\/td>\n<td>\u52a03\u4e2a\u6708<\/td>\n<\/tr>\n<tr>\n<td>last_day<\/td>\n<td>last_day(col(&#034;date&#034;))<\/td>\n<td>\u5f53\u6708\u6700\u540e\u4e00\u5929<\/td>\n<\/tr>\n<tr>\n<td>trunc<\/td>\n<td>trunc(col(&#034;date&#034;), &#034;MM&#034;)<\/td>\n<td>\u622a\u65ad\u5230\u6708\u521d<\/td>\n<\/tr>\n<tr>\n<td>unix_timestamp<\/td>\n<td>unix_timestamp(col(&#034;ts&#034;))<\/td>\n<td>\u8f6c\u4e3aUnix\u79d2\u6570<\/td>\n<\/tr>\n<tr>\n<td>from_unixtime<\/td>\n<td>from_unixtime(col(&#034;unix&#034;))<\/td>\n<td>\u79d2\u6570\u8f6c\u5b57\u7b26\u4e32<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u6ce8\u610f&#xff1a;to_date\u548cto_timestamp\u5728\u683c\u5f0f\u4e0d\u5339\u914d\u65f6\u4f1a\u8fd4\u56denull&#xff0c;\u53ef\u4ee5\u901a\u8fc7cast\u6216try_cast&#xff08;Spark 3.0&#043;&#xff09;\u5904\u7406\u3002<\/p>\n<h4>5.3 \u805a\u5408\u51fd\u6570\u8be6\u89e3<\/h4>\n<table>\n<tr>\u51fd\u6570\u793a\u4f8b\u8bf4\u660e<\/tr>\n<tbody>\n<tr>\n<td>\u5e38\u89c4\u805a\u5408<\/td>\n<td>sum, avg, count, max, min<\/td>\n<td>\u914d\u5408groupBy\u4f7f\u7528<\/td>\n<\/tr>\n<tr>\n<td>collect_list<\/td>\n<td>collect_list(col(&#034;name&#034;))<\/td>\n<td>\u6536\u96c6\u4e3a\u6570\u7ec4&#xff08;\u4fdd\u7559\u987a\u5e8f&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>collect_set<\/td>\n<td>collect_set(col(&#034;city&#034;))<\/td>\n<td>\u6536\u96c6\u4e3a\u6570\u7ec4&#xff08;\u53bb\u91cd&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>first\/last<\/td>\n<td>first(col(&#034;value&#034;))<\/td>\n<td>\u7b2c\u4e00\u4e2a\/\u6700\u540e\u4e00\u4e2a\u503c&#xff08;\u53ef\u80fd\u4e0d\u786e\u5b9a\u987a\u5e8f&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>approx_count_distinct<\/td>\n<td>approx_count_distinct(col(&#034;user_id&#034;), 0.05)<\/td>\n<td>\u8fd1\u4f3c\u53bb\u91cd\u8ba1\u6570&#xff08;HyperLogLog&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>grouping<\/td>\n<td>grouping(col(&#034;year&#034;))<\/td>\n<td>\u7528\u4e8eGROUP BY CUBE\/ROLLUP\u5224\u65ad\u662f\u5426\u4e3a\u805a\u5408\u884c<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>5.4 \u6761\u4ef6\u51fd\u6570\u8be6\u89e3<\/h4>\n<ul>\n<li>when(condition, value).otherwise(default)&#xff1a;\u7c7b\u4f3cCASE WHEN&#xff0c;\u591a\u4e2awhen\u53ef\u94fe\u5f0f\u8c03\u7528\u3002<\/li>\n<li>coalesce(col1, col2, &#8230;)&#xff1a;\u8fd4\u56de\u7b2c\u4e00\u4e2a\u975e\u7a7a\u503c\u3002<\/li>\n<li>isnull(col)&#xff1a;\u5224\u65ad\u662f\u5426\u4e3a\u7a7a&#xff0c;\u8fd4\u56de\u5e03\u5c14\u503c\u3002<\/li>\n<li>if(cond, true_val, false_val)&#xff1a;\u7c7b\u4f3c\u4e09\u5143\u8fd0\u7b97\u7b26\u3002<\/li>\n<li>nullif(expr1, expr2)&#xff1a;\u5982\u679c\u76f8\u7b49\u8fd4\u56deNULL&#xff0c;\u5426\u5219\u8fd4\u56deexpr1\u3002<\/li>\n<li>nanvl(col1, col2)&#xff1a;\u5982\u679ccol1\u662fNaN\u5219\u8fd4\u56decol2&#xff0c;\u5426\u5219col1\u3002<\/li>\n<\/ul>\n<h4>5.5 \u7a97\u53e3\u51fd\u6570\u8be6\u89e3<\/h4>\n<p>\u7a97\u53e3\u51fd\u6570\u8bed\u6cd5&#xff1a;\u51fd\u6570().over(Window.partitionBy(&#034;col1&#034;).orderBy(&#034;col2&#034;))<\/p>\n<p>\u5e38\u7528\u7a97\u53e3\u51fd\u6570&#xff1a;<\/p>\n<ul>\n<li>row_number()&#xff1a;\u8fde\u7eed\u552f\u4e00\u7f16\u53f7&#xff0c;\u4ece1\u5f00\u59cb\u3002<\/li>\n<li>rank()&#xff1a;\u6709\u95f4\u9694\u6392\u540d&#xff08;1,1,3,\u2026&#xff09;\u3002<\/li>\n<li>dense_rank()&#xff1a;\u65e0\u95f4\u9694\u6392\u540d&#xff08;1,1,2,\u2026&#xff09;\u3002<\/li>\n<li>percent_rank()&#xff1a;\u76f8\u5bf9\u6392\u540d (rank-1)\/(\u603b\u884c\u6570-1)\u3002<\/li>\n<li>ntile(n)&#xff1a;\u5c06\u5206\u533a\u5185\u6570\u636e\u5206\u6210n\u4e2a\u6876&#xff0c;\u8fd4\u56de\u6876\u7f16\u53f7\u3002<\/li>\n<li>lag(col, offset, default)&#xff1a;\u5411\u524d\u504f\u79fboffset\u884c\u3002<\/li>\n<li>lead(col, offset, default)&#xff1a;\u5411\u540e\u504f\u79fb\u3002<\/li>\n<li>\u805a\u5408\u51fd\u6570\u5982sum(col).over(&#8230;)&#xff1a;\u8ba1\u7b97\u7d2f\u8ba1\u548c\u3001\u79fb\u52a8\u5e73\u5747\u7b49\u3002<\/li>\n<\/ul>\n<p>\u7a97\u53e3\u8303\u56f4\u5b9a\u4e49&#xff1a;<\/p>\n<p><span class=\"token keyword\">from<\/span> pyspark<span class=\"token punctuation\">.<\/span>sql <span class=\"token keyword\">import<\/span> Window<br \/>\nWindow<span class=\"token punctuation\">.<\/span>partitionBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;user&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>orderBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;date&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>rowsBetween<span class=\"token punctuation\">(<\/span>Window<span class=\"token punctuation\">.<\/span>unboundedPreceding<span class=\"token punctuation\">,<\/span> Window<span class=\"token punctuation\">.<\/span>currentRow<span class=\"token punctuation\">)<\/span><\/p>\n<ul>\n<li>rowsBetween(start, end)&#xff1a;\u6309\u884c\u6570\u5b9a\u4e49\u7a97\u53e3\u8303\u56f4\u3002<\/li>\n<li>rangeBetween(start, end)&#xff1a;\u6309\u6392\u5e8f\u503c\u7684\u5dee\u503c\u5b9a\u4e49\u3002<\/li>\n<\/ul>\n<h4>5.6 \u96c6\u5408\u51fd\u6570&#xff08;\u6570\u7ec4\u4e0eMap&#xff09;<\/h4>\n<p>\u5904\u7406ArrayType\u5217\u7684\u51fd\u6570&#xff1a;<\/p>\n<ul>\n<li>array_contains(array, value)&#xff1a;\u662f\u5426\u5305\u542b\u3002<\/li>\n<li>size(array)&#xff1a;\u6570\u7ec4\u957f\u5ea6\u3002<\/li>\n<li>explode(array)&#xff1a;\u5c06\u6570\u7ec4\u6bcf\u4e2a\u5143\u7d20\u5c55\u5f00\u6210\u591a\u884c&#xff08;\u4e00\u5bf9\u591a&#xff09;\u3002<\/li>\n<li>posexplode(array)&#xff1a;\u540c\u65f6\u5c55\u5f00\u7d22\u5f15\u548c\u5143\u7d20\u3002<\/li>\n<li>array_distinct(array)&#xff1a;\u53bb\u91cd\u3002<\/li>\n<li>array_remove(array, value)&#xff1a;\u5220\u9664\u6307\u5b9a\u5143\u7d20\u3002<\/li>\n<li>array_union(array1, array2)&#xff1a;\u5e76\u96c6\u3002<\/li>\n<li>array_intersect(array1, array2)&#xff1a;\u4ea4\u96c6\u3002<\/li>\n<li>array_join(array, delimiter)&#xff1a;\u5c06\u6570\u7ec4\u5143\u7d20\u62fc\u63a5\u4e3a\u5b57\u7b26\u4e32\u3002<\/li>\n<\/ul>\n<p>\u5904\u7406MapType\u5217\u7684\u51fd\u6570&#xff1a;<\/p>\n<ul>\n<li>map_keys(map)&#xff1a;\u8fd4\u56de\u6240\u6709\u952e\u7684\u6570\u7ec4\u3002<\/li>\n<li>map_values(map)&#xff1a;\u8fd4\u56de\u6240\u6709\u503c\u7684\u6570\u7ec4\u3002<\/li>\n<li>map_from_arrays(keys, values)&#xff1a;\u4ece\u4e24\u4e2a\u6570\u7ec4\u521b\u5efaMap\u3002<\/li>\n<\/ul>\n<h3>\u516d\u3001\u6613\u9519\u70b9\u907f\u5751<\/h3>\n<h4>6.1 \u6df7\u6dc6substring\u7684\u7d22\u5f15\u8d77\u59cb<\/h4>\n<p>SQL\u4e2dsubstring(str, pos, len)\u7684pos\u4ece1\u5f00\u59cb&#xff0c;\u800cPython\u5b57\u7b26\u4e32\u5207\u7247\u4ece0\u5f00\u59cb\u3002\u9519\u8bef\u4e60\u60ef\u4f1a\u5bfc\u81f4\u622a\u53d6\u7ed3\u679c\u9519\u8bef\u3002<\/p>\n<h4>6.2 \u65e5\u671f\u683c\u5f0f\u5b57\u7b26\u4e32\u5927\u5c0f\u5199\u654f\u611f<\/h4>\n<p>date_format\u4e2d\u683c\u5f0f\u5fc5\u987b\u4e25\u683c\u5339\u914d&#xff1a;yyyy-MM-dd&#xff08;\u5c0f\u5199y\u8868\u793a\u5e74&#xff0c;\u5927\u5199M\u8868\u793a\u6708&#xff0c;\u5c0f\u5199m\u8868\u793a\u5206\u949f&#xff09;\u3002\u5e38\u89c1\u9519\u8bef&#xff1a;\u7528YYYY&#xff08;\u8fd9\u662f\u5468\u5e74\u5468\u6570&#xff09;\u5bfc\u81f4\u5e74\u4efd\u9519\u8bef\u3002<\/p>\n<h4>6.3 \u7a97\u53e3\u51fd\u6570\u4e2dorder by\u7f3a\u5c11\u4f1a\u5bfc\u81f4\u6574\u4e2a\u5206\u533a\u4f5c\u4e3a\u4e00\u4e2a\u7a97\u53e3<\/h4>\n<p>\u5982\u679c\u4e0d\u6307\u5b9aorderBy&#xff0c;\u7a97\u53e3\u9ed8\u8ba4\u4e3a\u6574\u4e2a\u5206\u533a&#xff08;\u65e0\u6ed1\u52a8\u7a97\u53e3&#xff09;\u3002\u4f46\u67d0\u4e9b\u7a97\u53e3\u51fd\u6570&#xff08;\u5982row_number&#xff09;\u8981\u6c42\u5fc5\u987b\u6307\u5b9aorderBy&#xff0c;\u5426\u5219\u62a5\u9519\u3002<\/p>\n<h4>6.4 collect_list\u7ed3\u679c\u65e0\u5e8f\u4e14\u53ef\u80fd\u53d7\u5206\u533a\u5f71\u54cd<\/h4>\n<p>collect_list\u4e0d\u4fdd\u8bc1\u987a\u5e8f&#xff0c;\u5982\u679c\u9700\u8981\u6709\u5e8f&#xff0c;\u5e94\u5148orderBy\u518d\u6536\u96c6\u3002<\/p>\n<h4>6.5 \u5728\u805a\u5408\u524d\u4f7f\u7528explode\u5bfc\u81f4\u6570\u636e\u81a8\u80c0<\/h4>\n<p>explode\u4f1a\u4e3a\u6570\u7ec4\u6bcf\u4e2a\u5143\u7d20\u751f\u6210\u4e00\u884c&#xff0c;\u5982\u679c\u6570\u7ec4\u5f88\u5927&#xff0c;\u4f1a\u5927\u5e45\u589e\u52a0\u6570\u636e\u91cf\u3002\u5728groupBy\u524d\u4f7f\u7528explode\u53ef\u80fd\u5bfc\u81f4Shuffle\u6570\u636e\u91cf\u7206\u70b8\u3002<\/p>\n<h4>6.6 \u4f7f\u7528when\u65f6\u9057\u6f0fotherwise<\/h4>\n<p>\u5982\u679c\u6ca1\u6709otherwise&#xff0c;\u4e0d\u6ee1\u8db3\u6761\u4ef6\u7684\u5206\u652f\u4f1a\u8fd4\u56denull&#xff0c;\u8fd9\u53ef\u80fd\u662f\u671f\u671b\u7684\u7ed3\u679c&#xff0c;\u4f46\u4e5f\u53ef\u80fd\u65e0\u610f\u4e2d\u5f15\u5165\u7a7a\u503c\u3002<\/p>\n<h4>6.7 coalesce\u4e0erepartition\u6df7\u6dc6<\/h4>\n<p>coalesce\u4f5c\u4e3a\u51fd\u6570\u7528\u4e8e\u8fd4\u56de\u7b2c\u4e00\u4e2a\u975e\u7a7a\u503c&#xff0c;\u800cDataFrame\u7684coalesce\u65b9\u6cd5\u7528\u4e8e\u51cf\u5c11\u5206\u533a\u3002\u6ce8\u610f\u4e0a\u4e0b\u6587\u3002<\/p>\n<h3>\u4e03\u3001\u5b8c\u6574\u5b9e\u6218\u6848\u4f8b<\/h3>\n<p>\u672c\u6848\u4f8b\u5c06\u4f7f\u7528\u4e00\u4e2a\u6a21\u62df\u7684\u7528\u6237\u884c\u4e3a\u65e5\u5fd7\u6570\u636e\u96c6&#xff0c;\u7efc\u5408\u8fd0\u7528\u5b57\u7b26\u4e32\u3001\u65e5\u671f\u3001\u805a\u5408\u3001\u6761\u4ef6\u3001\u7a97\u53e3\u3001\u96c6\u5408\u7b49\u5185\u7f6e\u51fd\u6570&#xff0c;\u5b8c\u6210\u4ee5\u4e0b\u5206\u6790\u4efb\u52a1&#xff1a;<\/p>\n<li>\u65e5\u5fd7\u89e3\u6790&#xff1a;\u4ece\u539f\u59cb\u5b57\u7b26\u4e32\u4e2d\u63d0\u53d6IP\u3001URL\u3001\u72b6\u6001\u7801\u3001\u54cd\u5e94\u65f6\u95f4<\/li>\n<li>\u6570\u636e\u6e05\u6d17&#xff1a;\u8f6c\u6362\u65e5\u671f\u683c\u5f0f\u3001\u5904\u7406\u7a7a\u503c<\/li>\n<li>\u7edf\u8ba1\u6bcf\u5c0f\u65f6\u7684PV\u3001UV\u3001\u5e73\u5747\u54cd\u5e94\u65f6\u95f4<\/li>\n<li>\u4f7f\u7528\u7a97\u53e3\u51fd\u6570\u8ba1\u7b97\u6bcf\u4e2aIP\u7684\u8bbf\u95ee\u6b21\u6570\u6392\u540d\u3001\u7d2f\u8ba1\u54cd\u5e94\u65f6\u95f4<\/li>\n<li>\u5bf9URL\u8fdb\u884c\u5206\u7c7b\u6807\u8bb0&#xff08;\u4f7f\u7528\u6761\u4ef6\u51fd\u6570&#xff09;<\/li>\n<li>\u5c06\u8bbf\u95ee\u8def\u5f84\u62c6\u5206\u4e3a\u6570\u7ec4\u5e76\u7edf\u8ba1\u8def\u5f84\u6df1\u5ea6\u5206\u5e03<\/li>\n<p><span class=\"token comment\"># &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061; builtin_functions_demo.py &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;<\/span><br \/>\n<span class=\"token comment\"># \u529f\u80fd&#xff1a;PySpark\u5185\u7f6e\u51fd\u6570\u5168\u89e3\u5b9e\u6218&#xff08;\u5b57\u7b26\u4e32\u3001\u65e5\u671f\u3001\u805a\u5408\u3001\u6761\u4ef6\u3001\u7a97\u53e3\u3001\u96c6\u5408&#xff09;<\/span><br \/>\n<span class=\"token comment\"># \u6570\u636e\u91cf&#xff1a;\u6a21\u62df100\u4e07\u6761Apache\u65e5\u5fd7<\/span><br \/>\n<span class=\"token comment\"># \u73af\u5883&#xff1a;PySpark 3.x<\/span><\/p>\n<p><span class=\"token keyword\">from<\/span> pyspark<span class=\"token punctuation\">.<\/span>sql <span class=\"token keyword\">import<\/span> SparkSession<br \/>\n<span class=\"token keyword\">from<\/span> pyspark<span class=\"token punctuation\">.<\/span>sql<span class=\"token punctuation\">.<\/span>functions <span class=\"token keyword\">import<\/span> <span class=\"token operator\">*<\/span><br \/>\n<span class=\"token keyword\">from<\/span> pyspark<span class=\"token punctuation\">.<\/span>sql<span class=\"token punctuation\">.<\/span>types <span class=\"token keyword\">import<\/span> <span class=\"token operator\">*<\/span><br \/>\n<span class=\"token keyword\">from<\/span> pyspark<span class=\"token punctuation\">.<\/span>sql <span class=\"token keyword\">import<\/span> Window<br \/>\n<span class=\"token keyword\">import<\/span> random<br \/>\n<span class=\"token keyword\">from<\/span> datetime <span class=\"token keyword\">import<\/span> datetime<span class=\"token punctuation\">,<\/span> timedelta<\/p>\n<p><span class=\"token comment\"># &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061; 1. \u521b\u5efaSparkSession &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;<\/span><br \/>\nspark <span class=\"token operator\">&#061;<\/span> SparkSession<span class=\"token punctuation\">.<\/span>builder \\\\<br \/>\n    <span class=\"token punctuation\">.<\/span>appName<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;BuiltinFunctionsDemo&#034;<\/span><span class=\"token punctuation\">)<\/span> \\\\<br \/>\n    <span class=\"token punctuation\">.<\/span>master<span class=\"token punctuation\">(<\/span>&#034;local<span class=\"token punctuation\">[<\/span><span class=\"token number\">4<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span> \\\\<br \/>\n    <span class=\"token punctuation\">.<\/span>config<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;spark.sql.shuffle.partitions&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;8&#034;<\/span><span class=\"token punctuation\">)<\/span> \\\\<br \/>\n    <span class=\"token punctuation\">.<\/span>getOrCreate<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>sc <span class=\"token operator\">&#061;<\/span> spark<span class=\"token punctuation\">.<\/span>sparkContext<br \/>\nsc<span class=\"token punctuation\">.<\/span>setLogLevel<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;WARN&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;PySpark \u5185\u7f6e\u51fd\u6570\u5168\u89e3\u5b9e\u6218&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061; 2. \u751f\u6210\u6a21\u62df\u65e5\u5fd7\u6570\u636e &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;<\/span><br \/>\n<span class=\"token comment\"># \u539f\u59cb\u65e5\u5fd7\u683c\u5f0f&#xff1a;IP &#8211; &#8211; [\u65e5\u671f:\u65f6\u95f4 \u65f6\u533a] &#034;METHOD URL PROTOCOL&#034; STATUS BYTES<\/span><br \/>\n<span class=\"token comment\"># \u793a\u4f8b&#xff1a;192.168.1.1 &#8211; &#8211; [01\/Jan\/2024:12:34:56 &#043;0800] &#034;GET \/api\/user HTTP\/1.1&#034; 200 1234<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\\\\n\u751f\u6210\u6a21\u62df\u8bbf\u95ee\u65e5\u5fd7&#8230;&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\nips <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;192.168.1.<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>i<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#034;<\/span><\/span> <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">101<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">]<\/span><br \/>\nurls <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token string\">&#034;\/api\/user&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\/api\/order&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\/api\/product&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\/home&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\/login&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\/static\/style.css&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\/api\/order\/create&#034;<\/span><span class=\"token punctuation\">]<\/span><br \/>\nmethods <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token string\">&#034;GET&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;POST&#034;<\/span><span class=\"token punctuation\">]<\/span><br \/>\nstatus_codes <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token number\">200<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">200<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">200<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">404<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">500<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">302<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">generate_log_line<\/span><span class=\"token punctuation\">(<\/span>timestamp<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    ip <span class=\"token operator\">&#061;<\/span> random<span class=\"token punctuation\">.<\/span>choice<span class=\"token punctuation\">(<\/span>ips<span class=\"token punctuation\">)<\/span><br \/>\n    method <span class=\"token operator\">&#061;<\/span> random<span class=\"token punctuation\">.<\/span>choice<span class=\"token punctuation\">(<\/span>methods<span class=\"token punctuation\">)<\/span><br \/>\n    url <span class=\"token operator\">&#061;<\/span> random<span class=\"token punctuation\">.<\/span>choice<span class=\"token punctuation\">(<\/span>urls<span class=\"token punctuation\">)<\/span><br \/>\n    status <span class=\"token operator\">&#061;<\/span> random<span class=\"token punctuation\">.<\/span>choice<span class=\"token punctuation\">(<\/span>status_codes<span class=\"token punctuation\">)<\/span><br \/>\n    bytes_sent <span class=\"token operator\">&#061;<\/span> random<span class=\"token punctuation\">.<\/span>randint<span class=\"token punctuation\">(<\/span><span class=\"token number\">100<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">10000<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    log_line <span class=\"token operator\">&#061;<\/span> <span class=\"token string-interpolation\"><span class=\"token string\">f&#039;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>ip<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> &#8211; &#8211; [<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>timestamp<span class=\"token punctuation\">.<\/span>strftime<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;%d\/%b\/%Y:%H:%M:%S &#043;0800&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">] &#034;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>method<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>url<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> HTTP\/1.1&#034; <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>status<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>bytes_sent<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#039;<\/span><\/span><br \/>\n    <span class=\"token keyword\">return<\/span> log_line<\/p>\n<p><span class=\"token comment\"># \u751f\u6210\u6700\u8fd17\u5929\u7684\u65e5\u5fd7&#xff0c;\u5171100\u4e07\u6761<\/span><br \/>\nnum_records <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">1_000_000<\/span><br \/>\nstart_date <span class=\"token operator\">&#061;<\/span> datetime<span class=\"token punctuation\">.<\/span>now<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span> timedelta<span class=\"token punctuation\">(<\/span>days<span class=\"token operator\">&#061;<\/span><span class=\"token number\">7<\/span><span class=\"token punctuation\">)<\/span><br \/>\nlogs <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><br \/>\n<span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>num_records<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    ts <span class=\"token operator\">&#061;<\/span> start_date <span class=\"token operator\">&#043;<\/span> timedelta<span class=\"token punctuation\">(<\/span>seconds<span class=\"token operator\">&#061;<\/span>random<span class=\"token punctuation\">.<\/span>randint<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">7<\/span><span class=\"token operator\">*<\/span><span class=\"token number\">24<\/span><span class=\"token operator\">*<\/span><span class=\"token number\">3600<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    logs<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span>generate_log_line<span class=\"token punctuation\">(<\/span>ts<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u521b\u5efaDataFrame&#xff08;\u5355\u5217&#xff09;<\/span><br \/>\nlog_df <span class=\"token operator\">&#061;<\/span> spark<span class=\"token punctuation\">.<\/span>createDataFrame<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">(<\/span>log<span class=\"token punctuation\">,<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">for<\/span> log <span class=\"token keyword\">in<\/span> logs<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token string\">&#034;raw_log&#034;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;\u751f\u6210\u4e86 <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>log_df<span class=\"token punctuation\">.<\/span>count<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">,<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> \u6761\u65e5\u5fd7\u8bb0\u5f55&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\nlog_df<span class=\"token punctuation\">.<\/span>show<span class=\"token punctuation\">(<\/span><span class=\"token number\">5<\/span><span class=\"token punctuation\">,<\/span> truncate<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061; 3. \u5b57\u7b26\u4e32\u51fd\u6570&#xff1a;\u89e3\u6790\u65e5\u5fd7 &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\\\\n&#034;<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u6b65\u9aa41: \u4f7f\u7528\u5b57\u7b26\u4e32\u51fd\u6570\u89e3\u6790\u65e5\u5fd7&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u5b9a\u4e49\u6b63\u5219\u8868\u8fbe\u5f0f\u63d0\u53d6\u5b57\u6bb5<\/span><br \/>\nparsed_df <span class=\"token operator\">&#061;<\/span> log_df<span class=\"token punctuation\">.<\/span>select<span class=\"token punctuation\">(<\/span><br \/>\n    regexp_extract<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;raw_log&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">r&#039;^(\\\\S&#043;)&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ip&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span>  <span class=\"token comment\"># IP<\/span><br \/>\n    regexp_extract<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;raw_log&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">r&#039;\\\\[(.*?)\\\\]&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;timestamp_str&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span>  <span class=\"token comment\"># \u65f6\u95f4\u6233<\/span><br \/>\n    regexp_extract<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;raw_log&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">r&#039;&#034;(\\\\S&#043;) (\\\\S&#043;) \\\\S&#043;&#034;&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;method&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span>  <span class=\"token comment\"># \u65b9\u6cd5<\/span><br \/>\n    regexp_extract<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;raw_log&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">r&#039;&#034;\\\\S&#043; (\\\\S&#043;) \\\\S&#043;&#034;&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;url&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span>  <span class=\"token comment\"># URL<\/span><br \/>\n    regexp_extract<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;raw_log&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">r&#039;&#034;\\\\S&#043; \\\\S&#043; \\\\S&#043;&#034; (\\\\d{3})&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>cast<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;int&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;status&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span>  <span class=\"token comment\"># \u72b6\u6001\u7801<\/span><br \/>\n    regexp_extract<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;raw_log&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">r&#039;&#034;\\\\S&#043; \\\\S&#043; \\\\S&#043;&#034; \\\\d{3} (\\\\d&#043;)&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>cast<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;int&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;bytes&#034;<\/span><span class=\"token punctuation\">)<\/span>  <span class=\"token comment\"># \u5b57\u8282\u6570<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u4f7f\u7528\u5b57\u7b26\u4e32\u51fd\u6570\u8fdb\u4e00\u6b65\u6e05\u6d17<\/span><br \/>\n<span class=\"token comment\"># \u53bb\u9664URL\u4e2d\u7684\u67e5\u8be2\u53c2\u6570&#xff08;?\u540e\u9762\u7684\u90e8\u5206&#xff09;<\/span><br \/>\nparsed_df <span class=\"token operator\">&#061;<\/span> parsed_df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;url_path&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    split<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;url&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\\\\\\\\?&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>getItem<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u5bf9IP\u8fdb\u884c\u5206\u7c7b&#xff08;IP\u6bb5&#xff09;<\/span><br \/>\nparsed_df <span class=\"token operator\">&#061;<\/span> parsed_df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;ip_segment&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    concat<span class=\"token punctuation\">(<\/span>split<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ip&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\\\\\\\\.&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>getItem<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> lit<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;.&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n           split<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ip&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\\\\\\\\.&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>getItem<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> lit<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;.&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n           split<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ip&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\\\\\\\\.&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>getItem<span class=\"token punctuation\">(<\/span><span class=\"token number\">2<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u89e3\u6790\u540e\u7684DataFrame:&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\nparsed_df<span class=\"token punctuation\">.<\/span>printSchema<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\nparsed_df<span class=\"token punctuation\">.<\/span>show<span class=\"token punctuation\">(<\/span><span class=\"token number\">10<\/span><span class=\"token punctuation\">,<\/span> truncate<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061; 4. \u65e5\u671f\u51fd\u6570&#xff1a;\u8f6c\u6362\u65f6\u95f4\u6233\u3001\u63d0\u53d6\u5c0f\u65f6\/\u661f\u671f\u51e0 &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\\\\n&#034;<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u6b65\u9aa42: \u4f7f\u7528\u65e5\u671f\u51fd\u6570\u5904\u7406\u65f6\u95f4&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u5c06\u5b57\u7b26\u4e32\u65f6\u95f4\u6233\u8f6c\u4e3atimestamp\u7c7b\u578b&#xff0c;\u683c\u5f0f&#xff1a;01\/Jan\/2024:12:34:56 &#043;0800<\/span><br \/>\n<span class=\"token comment\"># \u9700\u8981\u5148\u53bb\u6389\u65f6\u533a\u90e8\u5206<\/span><br \/>\nparsed_with_date <span class=\"token operator\">&#061;<\/span> parsed_df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;timestamp_clean&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    regexp_replace<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;timestamp_str&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034; \\\\\\\\&#043;\\\\\\\\d{4}$&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;log_time&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    to_timestamp<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;timestamp_clean&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;dd\/MMM\/yyyy:HH:mm:ss&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;date&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    to_date<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;log_time&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;hour&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    hour<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;log_time&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;day_of_week&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    dayofweek<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;log_time&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span>  <span class=\"token comment\"># 1&#061;Sunday, 7&#061;Saturday<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;weekday&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    when<span class=\"token punctuation\">(<\/span>dayofweek<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;log_time&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>isin<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">7<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;Weekend&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>otherwise<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;Weekday&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u65e5\u671f\u5904\u7406\u540e\u7684Schema:&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\nparsed_with_date<span class=\"token punctuation\">.<\/span>printSchema<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\nparsed_with_date<span class=\"token punctuation\">.<\/span>select<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;timestamp_str&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;log_time&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;date&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;hour&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;weekday&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>show<span class=\"token punctuation\">(<\/span><span class=\"token number\">5<\/span><span class=\"token punctuation\">,<\/span> truncate<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061; 5. \u6761\u4ef6\u51fd\u6570&#xff1a;\u72b6\u6001\u7801\u5206\u7c7b\u3001\u7a7a\u503c\u5904\u7406 &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\\\\n&#034;<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u6b65\u9aa43: \u4f7f\u7528\u6761\u4ef6\u51fd\u6570 (when\/otherwise, coalesce)&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u6839\u636e\u72b6\u6001\u7801\u5206\u7c7b<\/span><br \/>\nclassified_df <span class=\"token operator\">&#061;<\/span> parsed_with_date<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;status_category&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    when<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;status&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&lt;<\/span> <span class=\"token number\">300<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;Success&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token punctuation\">.<\/span>when<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;status&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&lt;<\/span> <span class=\"token number\">400<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;Redirect&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token punctuation\">.<\/span>when<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;status&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&lt;<\/span> <span class=\"token number\">500<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;Client Error&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token punctuation\">.<\/span>otherwise<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;Server Error&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;is_error&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    when<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;status&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&gt;&#061;<\/span> <span class=\"token number\">400<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token boolean\">True<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>otherwise<span class=\"token punctuation\">(<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u5904\u7406\u53ef\u80fd\u4e3a\u7a7a\u7684bytes\u5b57\u6bb5&#xff08;\u4f7f\u7528coalesce&#xff09;<\/span><br \/>\nclassified_df <span class=\"token operator\">&#061;<\/span> classified_df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;bytes_sent&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    coalesce<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;bytes&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> lit<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u6761\u4ef6\u5206\u7c7b\u7ed3\u679c:&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\nclassified_df<span class=\"token punctuation\">.<\/span>select<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;status&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;status_category&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;is_error&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;bytes_sent&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>show<span class=\"token punctuation\">(<\/span><span class=\"token number\">10<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061; 6. \u805a\u5408\u51fd\u6570 &#043; \u5206\u7ec4 &#043; \u900f\u89c6 &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\\\\n&#034;<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u6b65\u9aa44: \u805a\u5408\u51fd\u6570\u4e0e\u900f\u89c6&#xff08;pivot&#xff09;&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u6309\u5c0f\u65f6\u7edf\u8ba1PV\u3001UV\u3001\u5e73\u5747\u54cd\u5e94\u65f6\u95f4<\/span><br \/>\nhourly_stats <span class=\"token operator\">&#061;<\/span> classified_df<span class=\"token punctuation\">.<\/span>groupBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;date&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;hour&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>agg<span class=\"token punctuation\">(<\/span><br \/>\n    count<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;*&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;pv&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    approx_count_distinct<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ip&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;uv&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    avg<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;bytes&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;avg_bytes&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>orderBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;date&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;hour&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u6bcf\u5c0f\u65f6\u8bbf\u95ee\u7edf\u8ba1:&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\nhourly_stats<span class=\"token punctuation\">.<\/span>show<span class=\"token punctuation\">(<\/span><span class=\"token number\">10<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u4f7f\u7528pivot&#xff1a;\u6309\u72b6\u6001\u7c7b\u522b\u7edf\u8ba1\u6bcf\u5929\u7684\u6570\u91cf<\/span><br \/>\npivot_df <span class=\"token operator\">&#061;<\/span> classified_df<span class=\"token punctuation\">.<\/span>groupBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;date&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>pivot<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;status_category&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token string\">&#034;Success&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;Redirect&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;Client Error&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;Server Error&#034;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>count<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u6bcf\u5929\u5404\u72b6\u6001\u7c7b\u522b\u6570\u91cf&#xff08;\u900f\u89c6\u8868&#xff09;:&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\npivot_df<span class=\"token punctuation\">.<\/span>show<span class=\"token punctuation\">(<\/span><span class=\"token number\">5<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u4f7f\u7528collect_list\u6536\u96c6\u6bcf\u4e2a\u5c0f\u65f6\u7684URL\u5217\u8868&#xff08;\u7528\u4e8e\u540e\u7eed\u5206\u6790&#xff09;<\/span><br \/>\nurl_list_per_hour <span class=\"token operator\">&#061;<\/span> classified_df<span class=\"token punctuation\">.<\/span>groupBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;date&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;hour&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>agg<span class=\"token punctuation\">(<\/span><br \/>\n    collect_list<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;url_path&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;urls&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u6bcf\u5c0f\u65f6URL\u5217\u8868&#xff08;collect_list\u793a\u4f8b&#xff09;:&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\nurl_list_per_hour<span class=\"token punctuation\">.<\/span>show<span class=\"token punctuation\">(<\/span><span class=\"token number\">3<\/span><span class=\"token punctuation\">,<\/span> truncate<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061; 7. \u7a97\u53e3\u51fd\u6570&#xff1a;\u6392\u540d\u3001\u7d2f\u8ba1\u3001\u6ede\u540e &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\\\\n&#034;<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u6b65\u9aa45: \u7a97\u53e3\u51fd\u6570\u5b9e\u6218&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u7a97\u53e3\u5b9a\u4e49&#xff1a;\u6309IP\u5206\u533a&#xff0c;\u6309\u65f6\u95f4\u6392\u5e8f<\/span><br \/>\nwindow_spec <span class=\"token operator\">&#061;<\/span> Window<span class=\"token punctuation\">.<\/span>partitionBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ip&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>orderBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;log_time&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u4e3a\u6bcf\u4e2aIP\u7684\u8bf7\u6c42\u6309\u65f6\u95f4\u7f16\u53f7<\/span><br \/>\ndf_with_row_num <span class=\"token operator\">&#061;<\/span> classified_df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;request_seq&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    row_number<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>over<span class=\"token punctuation\">(<\/span>window_spec<span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u6bcf\u4e2aIP\u7684\u7d2f\u8ba1\u54cd\u5e94\u5b57\u8282\u6570&#xff08;\u7d2f\u8ba1\u548c&#xff09;<\/span><br \/>\ndf_with_cumsum <span class=\"token operator\">&#061;<\/span> classified_df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;cumulative_bytes&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token builtin\">sum<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;bytes&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>over<span class=\"token punctuation\">(<\/span>window_spec<span class=\"token punctuation\">.<\/span>rowsBetween<span class=\"token punctuation\">(<\/span>Window<span class=\"token punctuation\">.<\/span>unboundedPreceding<span class=\"token punctuation\">,<\/span> Window<span class=\"token punctuation\">.<\/span>currentRow<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u6bcf\u4e2aIP\u4e0a\u4e00\u6b21\u8bf7\u6c42\u7684\u65f6\u95f4&#xff08;LAG&#xff09;<\/span><br \/>\ndf_with_lag <span class=\"token operator\">&#061;<\/span> classified_df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;prev_log_time&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    lag<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;log_time&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>over<span class=\"token punctuation\">(<\/span>window_spec<span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;time_since_last&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    datediff<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;log_time&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;prev_log_time&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u6bcf\u4e2aIP\u7684\u8bf7\u6c42\u6b21\u6570\u6392\u540d&#xff08;\u5168\u5c40\u7a97\u53e3&#xff09;<\/span><br \/>\nip_rank_window <span class=\"token operator\">&#061;<\/span> Window<span class=\"token punctuation\">.<\/span>orderBy<span class=\"token punctuation\">(<\/span>desc<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;cnt&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\nip_rank_df <span class=\"token operator\">&#061;<\/span> classified_df<span class=\"token punctuation\">.<\/span>groupBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ip&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>count<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;rank&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    rank<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>over<span class=\"token punctuation\">(<\/span>ip_rank_window<span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span><span class=\"token builtin\">filter<\/span><span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;rank&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&lt;&#061;<\/span> <span class=\"token number\">10<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u8bbf\u95ee\u6b21\u6570Top10\u7684IP:&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\nip_rank_df<span class=\"token punctuation\">.<\/span>show<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u4f7f\u7528ntile\u5c06\u6bcf\u4e2aIP\u7684\u8bf7\u6c42\u5206\u4e3a5\u4e2a\u6876&#xff08;\u7528\u4e8e\u767e\u5206\u4f4d\u6570\u5206\u6790&#xff09;<\/span><br \/>\ndf_with_ntile <span class=\"token operator\">&#061;<\/span> classified_df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;bytes_ntile&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    ntile<span class=\"token punctuation\">(<\/span><span class=\"token number\">5<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>over<span class=\"token punctuation\">(<\/span>Window<span class=\"token punctuation\">.<\/span>partitionBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ip&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>orderBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;bytes&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u6bcf\u4e2aIP\u7684\u8bf7\u6c42\u6309\u54cd\u5e94\u5b57\u8282\u6570\u5206\u6876:&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\ndf_with_ntile<span class=\"token punctuation\">.<\/span>select<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ip&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;bytes&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;bytes_ntile&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>show<span class=\"token punctuation\">(<\/span><span class=\"token number\">20<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061; 8. \u96c6\u5408\u51fd\u6570&#xff08;\u6570\u7ec4\/Map&#xff09; &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\\\\n&#034;<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u6b65\u9aa46: \u96c6\u5408\u51fd\u6570&#xff08;\u6570\u7ec4\u64cd\u4f5c&#xff09;&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u5c06URL\u8def\u5f84\u6309&#039;\/&#039;\u62c6\u5206\u6210\u6570\u7ec4&#xff0c;\u5e76\u5206\u6790\u8def\u5f84\u6df1\u5ea6<\/span><br \/>\ndf_with_path <span class=\"token operator\">&#061;<\/span> classified_df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;path_parts&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    split<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;url_path&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\/&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;path_depth&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    size<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;path_parts&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span> <span class=\"token number\">1<\/span>  <span class=\"token comment\"># \u51cf\u53bb\u7a7a\u5b57\u7b26\u4e32<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;first_level&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    when<span class=\"token punctuation\">(<\/span>size<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;path_parts&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&gt;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;path_parts&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>getItem<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>otherwise<span class=\"token punctuation\">(<\/span>lit<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;root&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u4f7f\u7528explode\u5c55\u5f00\u8def\u5f84\u7ec4\u4ef6&#xff0c;\u7edf\u8ba1\u6bcf\u4e2a\u8def\u5f84\u7ec4\u4ef6\u7684\u8bbf\u95ee\u9891\u7387<\/span><br \/>\nexploded_df <span class=\"token operator\">&#061;<\/span> classified_df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;path_parts_array&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    split<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;url_path&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\/&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;part&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    explode<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;path_parts_array&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span><span class=\"token builtin\">filter<\/span><span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;part&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">!&#061;<\/span> <span class=\"token string\">&#034;&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>path_component_stats <span class=\"token operator\">&#061;<\/span> exploded_df<span class=\"token punctuation\">.<\/span>groupBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;part&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>count<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>orderBy<span class=\"token punctuation\">(<\/span>desc<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;count&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;URL\u8def\u5f84\u7ec4\u4ef6\u8bbf\u95ee\u9891\u7387&#xff08;Top10&#xff09;:&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\npath_component_stats<span class=\"token punctuation\">.<\/span>show<span class=\"token punctuation\">(<\/span><span class=\"token number\">10<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u4f7f\u7528array_contains\u5224\u65adURL\u662f\u5426\u5305\u542b\u7279\u5b9a\u5b57\u7b26\u4e32<\/span><br \/>\nclassified_df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;is_api_call&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    array_contains<span class=\"token punctuation\">(<\/span>split<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;url_path&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\/&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;api&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>select<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;url_path&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;is_api_call&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>show<span class=\"token punctuation\">(<\/span><span class=\"token number\">5<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u6536\u96c6\u6bcf\u4e2aIP\u8bbf\u95ee\u7684\u6240\u6709URL&#xff08;\u53bb\u91cd&#xff09;\u5e76\u62fc\u63a5<\/span><br \/>\nip_unique_urls <span class=\"token operator\">&#061;<\/span> classified_df<span class=\"token punctuation\">.<\/span>groupBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ip&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>agg<span class=\"token punctuation\">(<\/span><br \/>\n    collect_set<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;url_path&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;unique_urls&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token string\">&#034;urls_string&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    array_join<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;unique_urls&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;, &#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u6bcf\u4e2aIP\u8bbf\u95ee\u7684\u4e0d\u540cURL&#xff08;\u524d10\u4e2aIP&#xff09;:&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\nip_unique_urls<span class=\"token punctuation\">.<\/span>show<span class=\"token punctuation\">(<\/span><span class=\"token number\">10<\/span><span class=\"token punctuation\">,<\/span> truncate<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061; 9. \u7efc\u5408\u6848\u4f8b&#xff1a;\u7528\u6237\u884c\u4e3a\u6f0f\u6597\u5206\u6790 &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\\\\n&#034;<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u6b65\u9aa47: \u7efc\u5408\u6848\u4f8b &#8211; \u9875\u9762\u8bbf\u95ee\u6f0f\u6597\u5206\u6790&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u5047\u8bbe\u6211\u4eec\u5173\u6ce8\u7684\u8def\u5f84\u987a\u5e8f&#xff1a;\/home -&gt; \/api\/product -&gt; \/api\/order\/create -&gt; \/api\/order\/pay<\/span><br \/>\nfunnel_steps <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token string\">&#034;\/home&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\/api\/product&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\/api\/order\/create&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\/api\/order\/pay&#034;<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p><span class=\"token comment\"># \u4e3a\u6bcf\u4e2aIP\u7684\u8bbf\u95ee\u8def\u5f84\u6309\u65f6\u95f4\u6392\u5e8f&#xff0c;\u5e76\u83b7\u53d6\u8def\u5f84\u5e8f\u5217<\/span><br \/>\nfunnel_df <span class=\"token operator\">&#061;<\/span> classified_df<span class=\"token punctuation\">.<\/span>select<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ip&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;log_time&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;url_path&#034;<\/span><span class=\"token punctuation\">)<\/span> \\\\<br \/>\n    <span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;rn&#034;<\/span><span class=\"token punctuation\">,<\/span> row_number<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>over<span class=\"token punctuation\">(<\/span>Window<span class=\"token punctuation\">.<\/span>partitionBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ip&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>orderBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;log_time&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u7b80\u5316\u6f0f\u6597\u8ba1\u7b97&#xff1a;\u7edf\u8ba1\u6bcf\u4e2aIP\u662f\u5426\u6309\u987a\u5e8f\u8bbf\u95ee\u4e86\u8fd9\u4e9b\u8def\u5f84&#xff08;\u4f7f\u7528\u7a97\u53e3\u51fd\u6570\u7684lag&#xff09;<\/span><br \/>\n<span class=\"token comment\"># \u66f4\u590d\u6742\u53ef\u4f7f\u7528pivot\u6216\u5e8f\u5217\u5339\u914d&#xff0c;\u8fd9\u91cc\u6f14\u793a\u601d\u8def<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u6f0f\u6597\u5206\u6790&#xff08;\u7b80\u5316&#xff09;: \u7edf\u8ba1\u6bcf\u4e2a\u6b65\u9aa4\u7684\u72ec\u7acb\u8bbf\u5ba2\u6570&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">for<\/span> i<span class=\"token punctuation\">,<\/span> step <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">enumerate<\/span><span class=\"token punctuation\">(<\/span>funnel_steps<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    step_count <span class=\"token operator\">&#061;<\/span> classified_df<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">filter<\/span><span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;url_path&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#061;&#061;<\/span> step<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>select<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ip&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>distinct<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>count<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;\u6b65\u9aa4<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>i<span class=\"token operator\">&#043;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">: <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>step<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> &#8211; \u8bbf\u5ba2\u6570 <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>step_count<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061; 10. \u6027\u80fd\u4f18\u5316&#xff1a;\u6bd4\u8f83UDF\u4e0e\u5185\u7f6e\u51fd\u6570 &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\\\\n&#034;<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u6b65\u9aa48: \u6027\u80fd\u5bf9\u6bd4 &#8211; \u5185\u7f6e\u51fd\u6570 vs UDF&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token keyword\">from<\/span> pyspark<span class=\"token punctuation\">.<\/span>sql<span class=\"token punctuation\">.<\/span>functions <span class=\"token keyword\">import<\/span> udf<br \/>\n<span class=\"token keyword\">import<\/span> time<\/p>\n<p><span class=\"token comment\"># UDF\u7248\u672c&#xff1a;\u63d0\u53d6URL\u7684\u7b2c\u4e00\u7ea7\u8def\u5f84<\/span><br \/>\n<span class=\"token keyword\">def<\/span> <span class=\"token function\">extract_first_level_udf<\/span><span class=\"token punctuation\">(<\/span>url<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token keyword\">if<\/span> <span class=\"token keyword\">not<\/span> url<span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> <span class=\"token string\">&#034;&#034;<\/span><br \/>\n    parts <span class=\"token operator\">&#061;<\/span> url<span class=\"token punctuation\">.<\/span>split<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\/&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">return<\/span> parts<span class=\"token punctuation\">[<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token keyword\">if<\/span> <span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>parts<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&gt;<\/span> <span class=\"token number\">1<\/span> <span class=\"token keyword\">else<\/span> <span class=\"token string\">&#034;&#034;<\/span><\/p>\n<p>extract_first_level <span class=\"token operator\">&#061;<\/span> udf<span class=\"token punctuation\">(<\/span>extract_first_level_udf<span class=\"token punctuation\">,<\/span> StringType<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u5185\u7f6e\u51fd\u6570\u7248\u672c<\/span><br \/>\nstart_time <span class=\"token operator\">&#061;<\/span> time<span class=\"token punctuation\">.<\/span>time<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\ndf_builtin <span class=\"token operator\">&#061;<\/span> classified_df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;first_level&#034;<\/span><span class=\"token punctuation\">,<\/span> split<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;url_path&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\/&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>getItem<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\ndf_builtin<span class=\"token punctuation\">.<\/span>count<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\nbuiltin_time <span class=\"token operator\">&#061;<\/span> time<span class=\"token punctuation\">.<\/span>time<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span> start_time<\/p>\n<p>start_time <span class=\"token operator\">&#061;<\/span> time<span class=\"token punctuation\">.<\/span>time<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\ndf_udf <span class=\"token operator\">&#061;<\/span> classified_df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;first_level&#034;<\/span><span class=\"token punctuation\">,<\/span> extract_first_level<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;url_path&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\ndf_udf<span class=\"token punctuation\">.<\/span>count<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\nudf_time <span class=\"token operator\">&#061;<\/span> time<span class=\"token punctuation\">.<\/span>time<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span> start_time<\/p>\n<p><span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;\u5185\u7f6e\u51fd\u6570\u8017\u65f6: <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>builtin_time<span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.2f<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> \u79d2&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;UDF\u8017\u65f6: <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>udf_time<span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.2f<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> \u79d2&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;\u5185\u7f6e\u51fd\u6570\u6bd4UDF\u5feb <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>udf_time<span class=\"token operator\">\/<\/span>builtin_time<span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.1f<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> \u500d&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061; 11. \u5c55\u793a\u5e38\u7528\u51fd\u6570\u793a\u4f8b\u6c47\u603b &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\\\\n&#034;<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u9644\u5f55: \u5e38\u7528\u51fd\u6570\u5feb\u901f\u53c2\u8003&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">80<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token triple-quoted-string string\">&#034;&#034;&#034;<br \/>\n\u5b57\u7b26\u4e32: concat, concat_ws, length, upper, lower, trim, substring, split, regexp_extract, regexp_replace<br \/>\n\u65e5\u671f: current_date, current_timestamp, to_date, to_timestamp, date_format, year, month, datediff, date_add<br \/>\n\u805a\u5408: sum, avg, count, max, min, collect_list, collect_set, approx_count_distinct<br \/>\n\u6761\u4ef6: when, otherwise, coalesce, isnull, if<br \/>\n\u7a97\u53e3: row_number, rank, dense_rank, lag, lead, ntile, sum().over()<br \/>\n\u96c6\u5408: explode, posexplode, size, array_contains, array_distinct, array_join<br \/>\n&#034;&#034;&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061; 12. \u6e05\u7406 &#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;&#061;<\/span><br \/>\nspark<span class=\"token punctuation\">.<\/span>stop<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\\\\n\u2705 \u5185\u7f6e\u51fd\u6570\u6f14\u793a\u5b8c\u6210&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<h3>\u516b\u3001\u4ee3\u7801\u9010\u884c\u89e3\u6790<\/h3>\n<h4>8.1 \u6b63\u5219\u63d0\u53d6\u5b57\u6bb5<\/h4>\n<p>regexp_extract<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;raw_log&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">r&#039;^(\\\\S&#043;)&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ip&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>\u4f7f\u7528\u6b63\u5219\u6355\u83b7\u7ec4\u63d0\u53d6IP&#xff0c;^(\\\\S&#043;)\u5339\u914d\u884c\u9996\u975e\u7a7a\u767d\u5b57\u7b26\u5e8f\u5217\u30021\u8868\u793a\u8fd4\u56de\u7b2c\u4e00\u4e2a\u6355\u83b7\u7ec4\u3002\u7c7b\u4f3c\u5730\u63d0\u53d6\u5176\u4ed6\u5b57\u6bb5\u3002\u6ce8\u610f\u72b6\u6001\u7801\u548c\u5b57\u8282\u6570\u5728\u63d0\u53d6\u540e\u4f7f\u7528.cast(&#034;int&#034;)\u8f6c\u6362\u4e3a\u6574\u578b\u3002<\/p>\n<h4>8.2 \u5b57\u7b26\u4e32\u5206\u5272\u4e0e\u53d6\u5143\u7d20<\/h4>\n<p>split<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;url&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\\\\\\\\?&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>getItem<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>split\u8fd4\u56de\u6570\u7ec4&#xff0c;getItem(0)\u53d6\u7b2c\u4e00\u4e2a\u5143\u7d20\u3002\u6ce8\u610f\u6b63\u5219\u8868\u8fbe\u5f0f\u4e2d\u7684?\u9700\u8981\u8f6c\u4e49\u3002<\/p>\n<h4>8.3 \u65e5\u671f\u8f6c\u6362<\/h4>\n<p>regexp_replace<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;timestamp_str&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034; \\\\\\\\&#043;\\\\\\\\d{4}$&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>\u5148\u53bb\u6389\u65f6\u533a\u90e8\u5206&#xff08;\u5982&#043;0800&#xff09;&#xff0c;\u7136\u540e\u4f7f\u7528to_timestamp\u6307\u5b9a\u683c\u5f0fdd\/MMM\/yyyy:HH:mm:ss\u3002\u6ce8\u610f\u6708\u4efd\u662fMMM&#xff08;\u82f1\u6587\u7f29\u5199&#xff09;\u3002hour\u3001dayofweek\u7b49\u63d0\u53d6\u90e8\u5206\u3002<\/p>\n<h4>8.4 \u6761\u4ef6\u5206\u7c7b<\/h4>\n<p>when<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;status&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&lt;<\/span> <span class=\"token number\">300<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;Success&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">.<\/span>when<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;status&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&lt;<\/span> <span class=\"token number\">400<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;Redirect&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">.<\/span><span class=\"token punctuation\">.<\/span><span class=\"token punctuation\">.<\/span><\/p>\n<p>\u591a\u4e2awhen\u94fe\u5f0f\u8c03\u7528&#xff0c;\u7c7b\u4f3cCASE WHEN\u3002otherwise\u53ef\u9009\u3002<\/p>\n<h4>8.5 \u8fd1\u4f3c\u53bb\u91cd\u8ba1\u6570<\/h4>\n<p>approx_count_distinct<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ip&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;uv&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>\u5bf9\u4e8e\u8d85\u5927\u57fa\u6570\u53bb\u91cd&#xff08;\u5982UV&#xff09;&#xff0c;\u4f7f\u7528approx_count_distinct\u53ef\u4ee5\u63d0\u9ad8\u901f\u5ea6&#xff0c;\u8bef\u5dee\u53ef\u63a5\u53d7&#xff08;\u9ed8\u8ba4\u4e3a5%&#xff09;\u3002<\/p>\n<h4>8.6 \u900f\u89c6&#xff08;pivot&#xff09;<\/h4>\n<p>pivot<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;status_category&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token string\">&#034;Success&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;Redirect&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">.<\/span><span class=\"token punctuation\">.<\/span><span class=\"token punctuation\">.<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>\u5c06\u884c\u8f6c\u5217&#xff0c;\u7b2c\u4e8c\u4e2a\u53c2\u6570\u6307\u5b9a\u5217\u503c\u5217\u8868&#xff08;\u53ef\u9009&#xff0c;\u4f46\u6307\u5b9a\u540e\u6027\u80fd\u66f4\u597d&#xff09;\u3002<\/p>\n<h4>8.7 \u7a97\u53e3\u51fd\u6570<\/h4>\n<p>Window<span class=\"token punctuation\">.<\/span>partitionBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ip&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>orderBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;log_time&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\nrow_number<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>over<span class=\"token punctuation\">(<\/span>window_spec<span class=\"token punctuation\">)<\/span><\/p>\n<p>\u4e3a\u6bcf\u4e2aIP\u7684\u8bf7\u6c42\u6309\u65f6\u95f4\u7f16\u53f7\u3002sum(&#034;bytes&#034;).over(window_spec.rowsBetween(Window.unboundedPreceding, Window.currentRow)) \u8ba1\u7b97\u7d2f\u8ba1\u548c\u3002<\/p>\n<h4>8.8 \u6570\u7ec4\u5c55\u5f00<\/h4>\n<p>explode<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;path_parts_array&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>\u5c06\u6570\u7ec4\u6bcf\u4e2a\u5143\u7d20\u5c55\u5f00\u6210\u4e00\u884c&#xff0c;\u5e38\u7528\u4e8e\u65e5\u5fd7\u89e3\u6790\u4e2d\u7684\u6807\u7b7e\u62c6\u5206\u3002<\/p>\n<h4>8.9 UDF vs \u5185\u7f6e\u51fd\u6570\u6027\u80fd\u5bf9\u6bd4<\/h4>\n<p>\u5185\u7f6e\u51fd\u6570\u76f4\u63a5\u64cd\u4f5cJVM\u6570\u636e&#xff0c;\u800cUDF\u9700\u8981\u5e8f\u5217\u5316\u548c\u8fdb\u7a0b\u95f4\u901a\u4fe1&#xff0c;\u8017\u65f6\u9ad8\u51fa\u6570\u5341\u500d\u3002\u56e0\u6b64\u5f3a\u70c8\u5efa\u8bae\u4f18\u5148\u4f7f\u7528\u5185\u7f6e\u51fd\u6570\u3002<\/p>\n<h3>\u4e5d\u3001\u4e1a\u52a1\u573a\u666f\u843d\u5730\u5e94\u7528<\/h3>\n<h4>9.1 \u573a\u666f\u4e00&#xff1a;\u7528\u6237\u6ce8\u518c\u65e5\u5fd7\u4e2d\u7684\u8bbe\u5907\u4fe1\u606f\u89e3\u6790<\/h4>\n<p>\u539f\u59cb\u65e5\u5fd7\u4e2d\u7528\u6237\u4ee3\u7406\u5b57\u7b26\u4e32&#xff08;User-Agent&#xff09;\u9700\u8981\u63d0\u53d6\u64cd\u4f5c\u7cfb\u7edf\u3001\u6d4f\u89c8\u5668\u7248\u672c\u3002\u4f7f\u7528regexp_extract\u548cwhen\u7ec4\u5408\u3002<\/p>\n<p>df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;os&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    when<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ua&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>rlike<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;Windows&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;Windows&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token punctuation\">.<\/span>when<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ua&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>rlike<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;Mac&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;macOS&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token punctuation\">.<\/span>when<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ua&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>rlike<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;Linux&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;Linux&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token punctuation\">.<\/span>otherwise<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;Other&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;browser&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    regexp_extract<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;ua&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;(Chrome|Firefox|Safari|Edge)&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><\/p>\n<h4>9.2 \u573a\u666f\u4e8c&#xff1a;\u7535\u5546\u8ba2\u5355\u7684RFM\u5206\u5c42<\/h4>\n<p>\u4f7f\u7528\u7a97\u53e3\u51fd\u6570\u5bf9\u7528\u6237\u8fdb\u884c\u6d88\u8d39\u91d1\u989d\u5206\u6876&#xff0c;\u8ba1\u7b97\u6bcf\u4e2a\u7528\u6237\u7684\u6392\u540d\u3002<\/p>\n<p>rfm_df <span class=\"token operator\">&#061;<\/span> order_df<span class=\"token punctuation\">.<\/span>groupBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;user_id&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>agg<span class=\"token punctuation\">(<\/span><br \/>\n    <span class=\"token builtin\">max<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;order_date&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;last_order&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    count<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;*&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;frequency&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token builtin\">sum<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;amount&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>alias<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;monetary&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;recency&#034;<\/span><span class=\"token punctuation\">,<\/span> datediff<span class=\"token punctuation\">(<\/span>current_date<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;last_order&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token comment\"># \u5206\u6876\u6253\u5206<\/span><br \/>\nrfm_df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;r_score&#034;<\/span><span class=\"token punctuation\">,<\/span> ntile<span class=\"token punctuation\">(<\/span><span class=\"token number\">5<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>over<span class=\"token punctuation\">(<\/span>Window<span class=\"token punctuation\">.<\/span>orderBy<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;recency&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>desc<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> \\\\<br \/>\n     <span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;f_score&#034;<\/span><span class=\"token punctuation\">,<\/span> ntile<span class=\"token punctuation\">(<\/span><span class=\"token number\">5<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>over<span class=\"token punctuation\">(<\/span>Window<span class=\"token punctuation\">.<\/span>orderBy<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;frequency&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> \\\\<br \/>\n     <span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;m_score&#034;<\/span><span class=\"token punctuation\">,<\/span> ntile<span class=\"token punctuation\">(<\/span><span class=\"token number\">5<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>over<span class=\"token punctuation\">(<\/span>Window<span class=\"token punctuation\">.<\/span>orderBy<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;monetary&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<h4>9.3 \u573a\u666f\u4e09&#xff1a;\u5b9e\u65f6\u6d41\u5904\u7406\u4e2d\u7684JSON\u89e3\u6790<\/h4>\n<p>\u4f7f\u7528from_json\u51fd\u6570\u89e3\u6790Kafka\u6d88\u606f\u4e2d\u7684JSON\u5b57\u7b26\u4e32\u3002<\/p>\n<p><span class=\"token keyword\">from<\/span> pyspark<span class=\"token punctuation\">.<\/span>sql<span class=\"token punctuation\">.<\/span>types <span class=\"token keyword\">import<\/span> StructType<span class=\"token punctuation\">,<\/span> StructField<span class=\"token punctuation\">,<\/span> StringType<span class=\"token punctuation\">,<\/span> IntegerType<\/p>\n<p>schema <span class=\"token operator\">&#061;<\/span> StructType<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">[<\/span>StructField<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;user_id&#034;<\/span><span class=\"token punctuation\">,<\/span> IntegerType<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> StructField<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;action&#034;<\/span><span class=\"token punctuation\">,<\/span> StringType<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><br \/>\ndf <span class=\"token operator\">&#061;<\/span> df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;json&#034;<\/span><span class=\"token punctuation\">,<\/span> from_json<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;value&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> schema<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>select<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;json.*&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<h4>9.4 \u573a\u666f\u56db&#xff1a;\u65e5\u5fd7\u8def\u5f84\u5206\u6790\u4e2d\u7684\u8def\u5f84\u62c6\u5206<\/h4>\n<p>\u5206\u6790\u7528\u6237\u8bbf\u95ee\u8def\u5f84&#xff0c;\u7edf\u8ba1\u6bcf\u4e2a\u9875\u9762\u7684\u6df1\u5ea6\u3002<\/p>\n<p>df<span class=\"token punctuation\">.<\/span>withColumn<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;depth&#034;<\/span><span class=\"token punctuation\">,<\/span> size<span class=\"token punctuation\">(<\/span>split<span class=\"token punctuation\">(<\/span>col<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;url&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;\/&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span> \\\\<br \/>\n  <span class=\"token punctuation\">.<\/span>groupBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;depth&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>count<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>orderBy<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;depth&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<h3>\u5341\u3001\u5e38\u89c1\u62a5\u9519\u6392\u67e5<\/h3>\n<h4>10.1 to_timestamp\u8fd4\u56denull<\/h4>\n<p>\u539f\u56e0&#xff1a;\u65e5\u671f\u5b57\u7b26\u4e32\u683c\u5f0f\u4e0d\u5339\u914d&#xff0c;\u6216\u65f6\u533a\u95ee\u9898\u3002<\/p>\n<p>\u89e3\u51b3&#xff1a;\u4f7f\u7528date_format\u5148\u8f6c\u6362\u4e3a\u6807\u51c6\u683c\u5f0f&#xff0c;\u6216\u5c1d\u8bd5\u591a\u79cd\u683c\u5f0f&#xff08;Spark 3.0&#043;\u652f\u6301to_timestamp\u81ea\u52a8\u89e3\u6790\u5e38\u89c1\u683c\u5f0f&#xff09;\u3002<\/p>\n<h4>10.2 \u7a97\u53e3\u51fd\u6570\u62a5\u9519Window function without frame<\/h4>\n<p>\u539f\u56e0&#xff1a;\u67d0\u4e9b\u51fd\u6570&#xff08;\u5982sum over\u7a97\u53e3&#xff09;\u9700\u8981\u6307\u5b9a\u8303\u56f4&#xff08;rowsBetween&#xff09;\u6216\u9ed8\u8ba4\u8303\u56f4&#xff0c;\u4f46\u82e5\u540c\u65f6\u4f7f\u7528order by\u800c\u6ca1\u6709\u6307\u5b9a\u8303\u56f4&#xff0c;Spark\u4f1a\u4f7f\u7528\u9ed8\u8ba4\u8303\u56f4&#xff08;range between unbounded preceding and current row&#xff09;&#xff0c;\u67d0\u4e9b\u60c5\u51b5\u4e0b\u53ef\u80fd\u51fa\u9519\u3002\u5bf9\u4e8e\u6392\u540d\u51fd\u6570\u4e0d\u9700\u8981\u8303\u56f4\u3002<\/p>\n<p>\u89e3\u51b3&#xff1a;\u5bf9\u4e8e\u805a\u5408\u7a97\u53e3\u51fd\u6570&#xff0c;\u663e\u5f0f\u6307\u5b9arowsBetween\u3002<\/p>\n<h4>10.3 regexp_extract\u63d0\u53d6\u4e0d\u5230\u503c\u8fd4\u56de\u7a7a\u5b57\u7b26\u4e32<\/h4>\n<p>\u539f\u56e0&#xff1a;\u6b63\u5219\u672a\u5339\u914d&#xff0c;\u8fd4\u56de\u7a7a\u5b57\u7b26\u4e32\u800c\u975enull\u3002<\/p>\n<p>\u89e3\u51b3&#xff1a;\u4f7f\u7528when\u5224\u65ad\u957f\u5ea6\u6216\u7ed3\u5408regexp\u68c0\u67e5\u3002<\/p>\n<h4>10.4 \u4f7f\u7528collect_list\u5bfc\u81f4\u5185\u5b58\u6ea2\u51fa<\/h4>\n<p>\u539f\u56e0&#xff1a;\u5206\u7ec4\u5185\u6570\u636e\u91cf\u592a\u5927&#xff0c;\u6570\u7ec4\u5360\u7528\u5185\u5b58\u8fc7\u5927\u3002<\/p>\n<p>\u89e3\u51b3&#xff1a;\u9650\u5236\u6bcf\u7ec4\u5927\u5c0f&#xff0c;\u6216\u4f7f\u7528approx_collect_list&#xff08;\u7b2c\u4e09\u65b9\u5e93&#xff09;&#xff0c;\u6216\u6539\u7528\u5176\u4ed6\u805a\u5408\u65b9\u5f0f\u3002<\/p>\n<h4>10.5 explode\u540e\u6570\u636e\u81a8\u80c0\u5bfc\u81f4Shuffle\u6570\u636e\u91cf\u5de8\u5927<\/h4>\n<p>\u539f\u56e0&#xff1a;\u6bcf\u884c\u6570\u7ec4\u957f\u5ea6\u5f88\u5927&#xff0c;explode\u540e\u884c\u6570\u4e58\u500d\u6570\u589e\u52a0\u3002<\/p>\n<p>\u89e3\u51b3&#xff1a;\u5728explode\u524d\u5148\u8fc7\u6ee4\u6389\u7a7a\u6570\u7ec4&#xff0c;\u6216\u8003\u8651\u5176\u4ed6\u8bbe\u8ba1\u907f\u514d\u7206\u70b8\u3002<\/p>\n<h3>\u5341\u4e00\u3001\u672c\u8282\u8bfe\u77e5\u8bc6\u70b9\u603b\u7ed3<\/h3>\n<h4>\u51fd\u6570\u5206\u7c7b\u901f\u67e5\u8868<\/h4>\n<table>\n<tr>\u7c7b\u522b\u5e38\u7528\u51fd\u6570\u4f7f\u7528\u573a\u666f<\/tr>\n<tbody>\n<tr>\n<td>\u5b57\u7b26\u4e32<\/td>\n<td>split, regexp_extract, concat, trim, substring<\/td>\n<td>\u65e5\u5fd7\u89e3\u6790\u3001\u6570\u636e\u6e05\u6d17\u3001\u683c\u5f0f\u5316<\/td>\n<\/tr>\n<tr>\n<td>\u65e5\u671f<\/td>\n<td>to_date, datediff, date_add, year\/month, date_format<\/td>\n<td>\u65f6\u95f4\u7a97\u53e3\u3001\u5e74\u9f84\u8ba1\u7b97\u3001\u65f6\u5e8f\u5206\u6790<\/td>\n<\/tr>\n<tr>\n<td>\u805a\u5408<\/td>\n<td>collect_list, collect_set, approx_count_distinct, pivot<\/td>\n<td>\u5217\u8868\u805a\u5408\u3001\u53bb\u91cd\u8ba1\u6570\u3001\u884c\u5217\u8f6c\u6362<\/td>\n<\/tr>\n<tr>\n<td>\u6761\u4ef6<\/td>\n<td>when, coalesce, isnull<\/td>\n<td>\u6570\u636e\u5206\u7c7b\u3001\u7a7a\u503c\u586b\u5145<\/td>\n<\/tr>\n<tr>\n<td>\u7a97\u53e3<\/td>\n<td>row_number, rank, lag, lead, sum().over()<\/td>\n<td>\u6392\u540d\u3001\u73af\u6bd4\u3001\u7d2f\u8ba1\u8ba1\u7b97\u3001TopN<\/td>\n<\/tr>\n<tr>\n<td>\u96c6\u5408<\/td>\n<td>explode, size, array_contains, array_join<\/td>\n<td>\u6807\u7b7e\u5c55\u5f00\u3001\u6570\u7ec4\u64cd\u4f5c<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>\u6027\u80fd\u4f18\u5316\u8981\u70b9<\/h4>\n<li>\u4f18\u5148\u4f7f\u7528\u5185\u7f6e\u51fd\u6570&#xff0c;\u907f\u514dUDF<\/li>\n<li>\u5bf9\u4e8e\u5927\u6570\u7ec4\u7684collect_list\u8981\u8c28\u614e<\/li>\n<li>\u7a97\u53e3\u51fd\u6570\u4f1a\u89e6\u53d1\u6392\u5e8f&#xff0c;\u53ef\u8003\u8651\u5206\u533a\u51cf\u5c11\u6570\u636e\u91cf<\/li>\n<li>approx_count_distinct\u9002\u5408\u7cbe\u786e\u5ea6\u8981\u6c42\u4e0d\u9ad8\u7684\u573a\u666f<\/li>\n<li>\u4f7f\u7528pivot\u65f6\u6307\u5b9a\u5217\u503c\u5217\u8868\u53ef\u4f18\u5316\u6027\u80fd<\/li>\n<h3>\u5341\u4e8c\u3001\u8bfe\u540e\u601d\u8003\u4f5c\u4e1a<\/h3>\n<h4>\u4f5c\u4e1a\u4e00&#xff1a;\u7406\u8bba\u7406\u89e3\u9898<\/h4>\n<li>\n<p>\u4e3a\u4ec0\u4e48regexp_extract\u6bd4\u4f7f\u7528split&#043;getItem\u5728\u67d0\u4e9b\u573a\u666f\u4e0b\u66f4\u7075\u6d3b&#xff1f;\u8bf7\u4e3e\u4f8b\u8bf4\u660e\u3002<\/p>\n<\/li>\n<li>\n<p>\u7a97\u53e3\u51fd\u6570\u4e2drowsBetween\u548crangeBetween\u7684\u533a\u522b\u662f\u4ec0\u4e48&#xff1f;\u4ec0\u4e48\u65f6\u5019\u5e94\u8be5\u7528rangeBetween&#xff1f;<\/p>\n<\/li>\n<li>\n<p>\u7b80\u8ff0collect_set\u548carray_distinct\u7684\u533a\u522b\u53ca\u9002\u7528\u573a\u666f\u3002<\/p>\n<\/li>\n<h4>\u4f5c\u4e1a\u4e8c&#xff1a;\u4ee3\u7801\u5b9e\u8df5\u9898<\/h4>\n<li>\n<p>\u4f7f\u7528\u5185\u7f6e\u51fd\u6570\u4ece\u4ee5\u4e0b\u5b57\u7b26\u4e32\u4e2d\u63d0\u53d6\u6709\u6548\u4fe1\u606f&#xff1a;<\/p>\n<ul>\n<li>\u7528\u6237\u4ee3\u7406&#xff1a;\u201cMozilla\/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit\/537.36\u201d<\/li>\n<li>\u63d0\u53d6\u64cd\u4f5c\u7cfb\u7edf\u3001\u6d4f\u89c8\u5668\u540d\u79f0\u548c\u7248\u672c<\/li>\n<li>\u4f7f\u7528regexp_extract\u548cwhen\u7ec4\u5408<\/li>\n<\/ul>\n<\/li>\n<li>\n<p>\u7ed9\u5b9a\u4e00\u4e2a\u8ba2\u5355\u8868&#xff08;order_id, user_id, amount, order_time&#xff09;&#xff0c;\u4f7f\u7528\u7a97\u53e3\u51fd\u6570\u8ba1\u7b97&#xff1a;<\/p>\n<ul>\n<li>\u6bcf\u4e2a\u7528\u6237\u7684\u7d2f\u8ba1\u6d88\u8d39\u91d1\u989d&#xff08;\u6309\u65f6\u95f4\u6392\u5e8f&#xff09;<\/li>\n<li>\u6bcf\u4e2a\u7528\u6237\u5f53\u524d\u8ba2\u5355\u91d1\u989d\u4e0e\u524d\u4e00\u7b14\u8ba2\u5355\u91d1\u989d\u7684\u5dee\u503c&#xff08;LAG&#xff09;<\/li>\n<li>\u6bcf\u4e2a\u7528\u6237\u6d88\u8d39\u91d1\u989d\u768430\u5929\u79fb\u52a8\u5e73\u5747&#xff08;\u7a97\u53e3\u8303\u56f4\u6309\u65f6\u95f4&#xff0c;\u4f7f\u7528rangeBetween&#xff09;<\/li>\n<\/ul>\n<\/li>\n<li>\n<p>\u4f7f\u7528explode\u548cpivot\u5b9e\u73b0\u6807\u7b7e\u5206\u6790&#xff1a;\u6709\u8868user_tags(user_id, tags)\u5176\u4e2dtags\u662f\u9017\u53f7\u5206\u9694\u7684\u5b57\u7b26\u4e32\u3002\u7edf\u8ba1\u6bcf\u4e2a\u6807\u7b7e\u7684\u7528\u6237\u6570&#xff0c;\u5e76\u8f93\u51fa\u6bcf\u4e2a\u7528\u6237\u7684\u6807\u7b7e\u5217\u8868&#xff08;\u6570\u7ec4\u5f62\u5f0f&#xff09;\u3002<\/p>\n<\/li>\n<h4>\u4f5c\u4e1a\u4e09&#xff1a;\u573a\u666f\u5e94\u7528\u9898<\/h4>\n<p>\u67d0\u8d44\u8bafApp\u7684\u7528\u6237\u70b9\u51fb\u65e5\u5fd7\u5305\u542b\u5b57\u6bb5&#xff1a;user_id, article_id, event_time, read_duration&#xff08;\u9605\u8bfb\u65f6\u957f\u79d2&#xff09;\u3002\u9700\u8981\u5206\u6790&#xff1a;<\/p>\n<ul>\n<li>\u7528\u6237\u9605\u8bfb\u65f6\u957f\u7684\u5206\u5e03&#xff08;\u6309\u5206\u949f\u5206\u6876&#xff09;<\/li>\n<li>\u6bcf\u4e2a\u7528\u6237\u6700\u540e\u4e00\u6b21\u9605\u8bfb\u7684\u6587\u7ae0ID&#xff08;\u4f7f\u7528\u7a97\u53e3\u51fd\u6570&#xff09;<\/li>\n<li>\u6bcf\u5929\u9605\u8bfb\u65f6\u957f\u6700\u957f\u7684\u524d10\u7bc7\u6587\u7ae0<\/li>\n<li>\u5c06\u6587\u7ae0ID\u5217\u8868\u6309\u7528\u6237\u805a\u5408&#xff0c;\u5e76\u8f93\u51fa\u6bcf\u4e2a\u7528\u6237\u9605\u8bfb\u8fc7\u7684\u4e0d\u540c\u6587\u7ae0\u6570&#xff08;\u53bb\u91cd&#xff09;<\/li>\n<\/ul>\n<p>\u8bf7\u4f7f\u7528\u5185\u7f6e\u51fd\u6570\u5b9e\u73b0\u4e0a\u8ff0\u5206\u6790&#xff0c;\u5e76\u8bf4\u660e\u5173\u952e\u6b65\u9aa4\u4f7f\u7528\u7684\u51fd\u6570\u3002<\/p>\n<h4>\u4f5c\u4e1a\u56db&#xff1a;\u62d3\u5c55\u7814\u7a76<\/h4>\n<li>\n<p>\u6df1\u5165\u7814\u7a76Spark 3.0&#043;\u5f15\u5165\u7684to_timestamp\u7684\u683c\u5f0f\u81ea\u52a8\u63a8\u65ad\u7279\u6027&#xff0c;\u6d4b\u8bd5\u4e0d\u540c\u683c\u5f0f\u5b57\u7b26\u4e32\u7684\u89e3\u6790\u6548\u679c\u3002<\/p>\n<\/li>\n<li>\n<p>\u6bd4\u8f83approx_count_distinct\u4e0e\u7cbe\u786ecount(distinct)\u5728\u767e\u4e07\u3001\u5343\u4e07\u3001\u4ebf\u7ea7\u6570\u636e\u91cf\u4e0a\u7684\u6027\u80fd\u548c\u8bef\u5dee&#xff0c;\u64b0\u5199\u5b9e\u9a8c\u62a5\u544a\u3002<\/p>\n<\/li>\n<li>\n<p>\u9605\u8bfbSpark\u6e90\u7801\u4e2dWindow\u7b97\u5b50\u7684\u5b9e\u73b0&#xff0c;\u5206\u6790\u5176\u5185\u5b58\u548c\u6392\u5e8f\u5f00\u9500&#xff0c;\u63d0\u51fa\u4f18\u5316\u7b56\u7565\u3002<\/p>\n<\/li>\n<hr \/>\n<p>\u63d0\u4ea4\u65b9\u5f0f&#xff1a;\u672c\u6b21\u4f5c\u4e1a\u8981\u6c42\u63d0\u4ea4\u4ee3\u7801\u811a\u672c\u548c\u8fd0\u884c\u7ed3\u679c\u622a\u56fe&#xff0c;\u4ee5\u53ca\u7406\u8bba\u9898\u7684\u7b54\u6848\u3002\u9f13\u52b1\u5c06\u5e38\u7528\u7684\u5185\u7f6e\u51fd\u6570\u5c01\u88c5\u6210\u5de5\u5177\u7c7b\u3002<\/p>\n<p>\u6269\u5c55\u9605\u8bfb&#xff1a;<\/p>\n<ul>\n<li>PySpark\u5b98\u65b9\u51fd\u6570\u6587\u6863&#xff1a;pyspark.sql.functions<\/li>\n<li>\u300aSpark SQL \u6027\u80fd\u4f18\u5316\u300b\u7b2c6\u7ae0<\/li>\n<li>\u5f00\u6e90\u9879\u76ee&#xff1a;pyspark-builtin-functions-cheatsheet<\/li>\n<\/ul>\n<p>\u901a\u8fc7\u672c\u8282\u8bfe\u7684\u5b66\u4e60&#xff0c;\u4f60\u5df2\u7ecf\u638c\u63e1\u4e86PySpark\u5185\u7f6e\u51fd\u6570\u7684\u7cbe\u9ad3&#xff0c;\u80fd\u591f\u9ad8\u6548\u5730\u8fdb\u884c\u6570\u636e\u8f6c\u6362\u548c\u5206\u6790\u3002\u5728\u5b9e\u9645\u5de5\u4f5c\u4e2d&#xff0c;\u9047\u5230\u6570\u636e\u5904\u7406\u9700\u6c42&#xff0c;\u5e94\u7b2c\u4e00\u65f6\u95f4\u60f3\u5230\u5185\u7f6e\u51fd\u6570&#xff0c;\u800c\u4e0d\u662fUDF\u3002\u4e0b\u4e00\u8282\u8bfe\u6211\u4eec\u5c06\u5b66\u4e60PySpark\u81ea\u5b9a\u4e49\u51fd\u6570&#xff08;UDF\u3001UDAF\u3001UDTF&#xff09;\u7684\u5f00\u53d1\u539f\u7406\u4e0e\u4f01\u4e1a\u5b9e\u6218&#xff0c;\u4e86\u89e3\u5728\u5fc5\u987b\u4f7f\u7528\u81ea\u5b9a\u4e49\u903b\u8f91\u65f6\u5982\u4f55\u5199\u51fa\u9ad8\u6027\u80fd\u7684\u6269\u5c55\u51fd\u6570\u3002\u6211\u4eec\u4e0b\u8282\u8bfe\u89c1&#xff01;<\/p>\n<hr \/>\n<h2>&#x1f517;\u300a20\u8282\u8bfe PySpark \u4ece\u5165\u95e8\u5230\u7cbe\u901a\u300b\u7cfb\u5217\u8bfe\u7a0b\u5bfc\u822a<\/h2>\n<p>\u53bb\u8ba2\u9605<\/p>\n<p>&#x1f31f; \u611f\u8c22\u60a8\u8010\u5fc3\u9605\u8bfb\u5230\u8fd9\u91cc&#xff01; &#x1f4a1; \u5982\u679c\u672c\u6587\u5bf9\u60a8\u6709\u6240\u542f\u53d1\u6b22\u8fce&#xff1a; &#x1f44d; \u70b9\u8d5e&#x1f4cc; \u6536\u85cf &#x1f4e4; \u5206\u4eab\u7ed9\u66f4\u591a\u9700\u8981\u7684\u4f19\u4f34\u3002 &#x1f5e3;\ufe0f \u671f\u5f85\u5728\u8bc4\u8bba\u533a\u770b\u5230\u60a8\u7684\u60f3\u6cd5, \u5171\u540c\u8fdb\u6b65\u3002 &#x1f514; \u5173\u6ce8\u6211&#xff0c;\u6301\u7eed\u83b7\u53d6\u66f4\u591a\u5e72\u8d27\u5185\u5bb9&#xff5e; &#x1f917; \u6211\u4eec\u4e0b\u7bc7\u6587\u7ae0\u89c1&#xff5e;<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u6587\u7ae0\u76ee\u5f55\u4e00\u3001\u8bfe\u524d\u5bfc\u8bfb\u4e8c\u3001\u5b66\u4e60\u76ee\u6807\u4e09\u3001\u6838\u5fc3\u7406\u8bba\u77e5\u8bc6\u70b9\u56db\u3001\u539f\u7406\u901a\u4fd7\u8bb2\u89e34.1 \u5185\u7f6e\u51fd\u6570\u4e3a\u4f55\u6bd4UDF\u5feb\u5f97\u591a&#xff1f;4.2 \u51fd\u6570\u7ec4\u5408\u4e0e\u94fe\u5f0f\u8c03\u75284.3 \u7a97\u53e3\u51fd\u6570\u7684\u7269\u7406\u6267\u884c\u4e94\u3001\u91cd\u70b9\u6982\u5ff5\u62c6\u89e35.1 \u5b57\u7b26\u4e32\u51fd\u6570\u8be6\u89e35.2 \u65e5\u671f\u65f6\u95f4\u51fd\u6570\u8be6\u89e35.3 \u805a\u5408\u51fd\u6570\u8be6\u89e35.4 \u6761\u4ef6\u51fd\u6570\u8be6\u89e35.5 \u7a97\u53e3\u51fd\u6570\u8be6\u89e35.6 \u96c6\u5408\u51fd\u6570&#xff08;\u6570\u7ec4\u4e0eMap&#xff09;\u516d\u3001\u6613\u9519\u70b9\u907f\u57516.1 \u6df7\u6dc6substring\u7684\u7d22\u5f15\u8d77\u59cb6.2 \u65e5\u671f\u683c\u5f0f\u5b57\u7b26\u4e32\u5927\u5c0f\u5199\u654f\u611f6.3 \u7a97\u53e3\u51fd\u6570\u4e2dorder by\u7f3a\u5c11\u4f1a\u5bfc\u81f4<\/p>\n","protected":false},"author":2,"featured_media":89274,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[10128,81],"topic":[],"class_list":["post-89275","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-server","tag-pyspark","tag-python"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v20.3 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>\u7b2c14\u8bfe\uff1aPySpark\uff5c\u5185\u7f6e\u51fd\u6570\u5168\u89e3\u3010\u5b57\u7b26\u4e32\u65e5\u671f\u805a\u5408\u6761\u4ef6\u7a97\u53e3\u51fd\u6570\u5b9e\u6218\u3011 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/89275.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u7b2c14\u8bfe\uff1aPySpark\uff5c\u5185\u7f6e\u51fd\u6570\u5168\u89e3\u3010\u5b57\u7b26\u4e32\u65e5\u671f\u805a\u5408\u6761\u4ef6\u7a97\u53e3\u51fd\u6570\u5b9e\u6218\u3011 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"\u6587\u7ae0\u76ee\u5f55\u4e00\u3001\u8bfe\u524d\u5bfc\u8bfb\u4e8c\u3001\u5b66\u4e60\u76ee\u6807\u4e09\u3001\u6838\u5fc3\u7406\u8bba\u77e5\u8bc6\u70b9\u56db\u3001\u539f\u7406\u901a\u4fd7\u8bb2\u89e34.1 \u5185\u7f6e\u51fd\u6570\u4e3a\u4f55\u6bd4UDF\u5feb\u5f97\u591a&#xff1f;4.2 \u51fd\u6570\u7ec4\u5408\u4e0e\u94fe\u5f0f\u8c03\u75284.3 \u7a97\u53e3\u51fd\u6570\u7684\u7269\u7406\u6267\u884c\u4e94\u3001\u91cd\u70b9\u6982\u5ff5\u62c6\u89e35.1 \u5b57\u7b26\u4e32\u51fd\u6570\u8be6\u89e35.2 \u65e5\u671f\u65f6\u95f4\u51fd\u6570\u8be6\u89e35.3 \u805a\u5408\u51fd\u6570\u8be6\u89e35.4 \u6761\u4ef6\u51fd\u6570\u8be6\u89e35.5 \u7a97\u53e3\u51fd\u6570\u8be6\u89e35.6 \u96c6\u5408\u51fd\u6570&#xff08;\u6570\u7ec4\u4e0eMap&#xff09;\u516d\u3001\u6613\u9519\u70b9\u907f\u57516.1 \u6df7\u6dc6substring\u7684\u7d22\u5f15\u8d77\u59cb6.2 \u65e5\u671f\u683c\u5f0f\u5b57\u7b26\u4e32\u5927\u5c0f\u5199\u654f\u611f6.3 \u7a97\u53e3\u51fd\u6570\u4e2dorder by\u7f3a\u5c11\u4f1a\u5bfc\u81f4\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/89275.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-08-02T11:17:07+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/08\/20260802111705-6a6f273153ea9.jpg\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"14 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/89275.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/89275.html\",\"name\":\"\u7b2c14\u8bfe\uff1aPySpark\uff5c\u5185\u7f6e\u51fd\u6570\u5168\u89e3\u3010\u5b57\u7b26\u4e32\u65e5\u671f\u805a\u5408\u6761\u4ef6\u7a97\u53e3\u51fd\u6570\u5b9e\u6218\u3011 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-08-02T11:17:07+00:00\",\"dateModified\":\"2026-08-02T11:17:07+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/89275.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/89275.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/89275.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u7b2c14\u8bfe\uff1aPySpark\uff5c\u5185\u7f6e\u51fd\u6570\u5168\u89e3\u3010\u5b57\u7b26\u4e32\u65e5\u671f\u805a\u5408\u6761\u4ef6\u7a97\u53e3\u51fd\u6570\u5b9e\u6218\u3011\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"\u7b2c14\u8bfe\uff1aPySpark\uff5c\u5185\u7f6e\u51fd\u6570\u5168\u89e3\u3010\u5b57\u7b26\u4e32\u65e5\u671f\u805a\u5408\u6761\u4ef6\u7a97\u53e3\u51fd\u6570\u5b9e\u6218\u3011 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/89275.html","og_locale":"zh_CN","og_type":"article","og_title":"\u7b2c14\u8bfe\uff1aPySpark\uff5c\u5185\u7f6e\u51fd\u6570\u5168\u89e3\u3010\u5b57\u7b26\u4e32\u65e5\u671f\u805a\u5408\u6761\u4ef6\u7a97\u53e3\u51fd\u6570\u5b9e\u6218\u3011 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"\u6587\u7ae0\u76ee\u5f55\u4e00\u3001\u8bfe\u524d\u5bfc\u8bfb\u4e8c\u3001\u5b66\u4e60\u76ee\u6807\u4e09\u3001\u6838\u5fc3\u7406\u8bba\u77e5\u8bc6\u70b9\u56db\u3001\u539f\u7406\u901a\u4fd7\u8bb2\u89e34.1 \u5185\u7f6e\u51fd\u6570\u4e3a\u4f55\u6bd4UDF\u5feb\u5f97\u591a&#xff1f;4.2 \u51fd\u6570\u7ec4\u5408\u4e0e\u94fe\u5f0f\u8c03\u75284.3 \u7a97\u53e3\u51fd\u6570\u7684\u7269\u7406\u6267\u884c\u4e94\u3001\u91cd\u70b9\u6982\u5ff5\u62c6\u89e35.1 \u5b57\u7b26\u4e32\u51fd\u6570\u8be6\u89e35.2 \u65e5\u671f\u65f6\u95f4\u51fd\u6570\u8be6\u89e35.3 \u805a\u5408\u51fd\u6570\u8be6\u89e35.4 \u6761\u4ef6\u51fd\u6570\u8be6\u89e35.5 \u7a97\u53e3\u51fd\u6570\u8be6\u89e35.6 \u96c6\u5408\u51fd\u6570&#xff08;\u6570\u7ec4\u4e0eMap&#xff09;\u516d\u3001\u6613\u9519\u70b9\u907f\u57516.1 \u6df7\u6dc6substring\u7684\u7d22\u5f15\u8d77\u59cb6.2 \u65e5\u671f\u683c\u5f0f\u5b57\u7b26\u4e32\u5927\u5c0f\u5199\u654f\u611f6.3 \u7a97\u53e3\u51fd\u6570\u4e2dorder by\u7f3a\u5c11\u4f1a\u5bfc\u81f4","og_url":"https:\/\/www.wsisp.com\/helps\/89275.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-08-02T11:17:07+00:00","og_image":[{"url":"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/08\/20260802111705-6a6f273153ea9.jpg"}],"author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"14 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/89275.html","url":"https:\/\/www.wsisp.com\/helps\/89275.html","name":"\u7b2c14\u8bfe\uff1aPySpark\uff5c\u5185\u7f6e\u51fd\u6570\u5168\u89e3\u3010\u5b57\u7b26\u4e32\u65e5\u671f\u805a\u5408\u6761\u4ef6\u7a97\u53e3\u51fd\u6570\u5b9e\u6218\u3011 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-08-02T11:17:07+00:00","dateModified":"2026-08-02T11:17:07+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/89275.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/89275.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/89275.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"\u7b2c14\u8bfe\uff1aPySpark\uff5c\u5185\u7f6e\u51fd\u6570\u5168\u89e3\u3010\u5b57\u7b26\u4e32\u65e5\u671f\u805a\u5408\u6761\u4ef6\u7a97\u53e3\u51fd\u6570\u5b9e\u6218\u3011"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/89275","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=89275"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/89275\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media\/89274"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=89275"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=89275"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=89275"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=89275"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}