digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (10)\n \nduration: 0 ms";
2 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 0 ms<br>number of output rows: 1"];
}
3 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 1<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 1<br>local bytes read: 59.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 16.0 B<br>local merged bytes read: 0.0 B<br>number of partitions: 1<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 59.0 B"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (9)\n \nduration: 42 ms";
5 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 41 ms<br>peak memory: 0.0 B<br>number of output rows: 1<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
6 [labelType="html" label="<br><b>Project</b><br><br>"];
7 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 4,253<br>spill size: 0.0 B"];
}
subgraph cluster8 {
isCluster="true";
label="WholeStageCodegen (7)\n \nduration: 92 ms";
9 [labelType="html" label="<b>Sort</b><br><br>sort time: 17 ms<br>peak memory: 65.0 MiB<br>spill size: 0.0 B"];
}
10 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 99.3 KiB<br>number of coalesced partitions: 1"];
11 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 24,850<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>17 ms (2 ms, 3 ms, 4 ms (stage 41.0: task 36))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 24,850<br>local bytes read: 94.3 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 5<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size total (min, med, max (stageId: taskId))<br>582.4 KiB (39.1 KiB, 116.7 KiB, 195.0 KiB (stage 41.0: task 33))<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>94.3 KiB (6.8 KiB, 18.8 KiB, 31.2 KiB (stage 41.0: task 33))"];
12 [labelType="html" label="<br><b>Union</b><br><br>"];
subgraph cluster13 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: 381 ms";
14 [labelType="html" label="<br><b>Project</b><br><br>"];
15 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 8,319"];
}
16 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 8,319<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 2.1 MiB"];
subgraph cluster17 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: 380 ms";
18 [labelType="html" label="<br><b>Project</b><br><br>"];
19 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 6,597"];
}
20 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 6,597<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 2.1 MiB"];
subgraph cluster21 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: 374 ms";
22 [labelType="html" label="<br><b>Project</b><br><br>"];
23 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 4,980"];
}
24 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 4,980<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 2.1 MiB"];
subgraph cluster25 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: 384 ms";
26 [labelType="html" label="<br><b>Project</b><br><br>"];
27 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 3,285"];
}
28 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 3,285<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 2.1 MiB"];
subgraph cluster29 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: 339 ms";
30 [labelType="html" label="<br><b>Project</b><br><br>"];
31 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 1,669"];
}
32 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 1,669<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 2.1 MiB"];
subgraph cluster33 {
isCluster="true";
label="WholeStageCodegen (8)\n \nduration: 0 ms";
34 [labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 64.1 MiB<br>spill size: 0.0 B"];
}
35 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 15.2 KiB<br>number of coalesced partitions: 1"];
36 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 2,022<br>local merged chunks fetched: 0<br>shuffle write time: 4 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 2,022<br>local bytes read: 14.5 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 47.4 KiB<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 14.5 KiB"];
subgraph cluster37 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: 130 ms";
38 [labelType="html" label="<br><b>Project</b><br><br>"];
39 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 2,022"];
}
40 [labelType="html" label="<b>Scan csv </b><br><br>number of output rows: 2,022<br>number of files read: 1<br>metadata time: 0 ms<br>size of files read: 196.4 KiB"];
2->0;
3->2;
5->3;
6->5;
7->6;
9->7;
10->9;
11->10;
12->11;
14->12;
15->14;
16->15;
18->12;
19->18;
20->19;
22->12;
23->22;
24->23;
26->12;
27->26;
28->27;
30->12;
31->30;
32->31;
34->7;
35->34;
36->35;
38->36;
39->38;
40->39;
}
41
AdaptiveSparkPlan isFinalPlan=true
HashAggregate(keys=[], functions=[count(1)])
WholeStageCodegen (10)
Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1429]
HashAggregate(keys=[], functions=[partial_count(1)])
Project
SortMergeJoin [COL_983A817F_3EA2_4B94_AD4A_A0B9FE6C2965#2755], [REGROUPEMENT_CO_14459_CODE_CIM10_14460#2791], Inner
WholeStageCodegen (9)
Sort [COL_983A817F_3EA2_4B94_AD4A_A0B9FE6C2965#2755 ASC NULLS FIRST], false, 0
WholeStageCodegen (7)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_983A817F_3EA2_4B94_AD4A_A0B9FE6C2965#2755, 10), ENSURE_REQUIREMENTS, [plan_id=1114]
Union
Project [Code_DA1#2018 AS COL_983A817F_3EA2_4B94_AD4A_A0B9FE6C2965#2755]
Filter isnotnull(Code_DA1#2018)
WholeStageCodegen (1)
FileScan csv [Code_DA1#2018] Batched: false, DataFilters: [isnotnull(Code_DA1#2018)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/execution/DIAGNOSTIC_14331/current/diagnost..., PartitionFilters: [], PushedFilters: [IsNotNull(Code_DA1)], ReadSchema: struct<Code_DA1:string>
Project [Code_DA2#2531 AS COL_983A817F_3EA2_4B94_AD4A_A0B9FE6C2965#3073]
Filter isnotnull(Code_DA2#2531)
WholeStageCodegen (2)
FileScan csv [Code_DA2#2531] Batched: false, DataFilters: [isnotnull(Code_DA2#2531)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/execution/DIAGNOSTIC_14331/current/diagnost..., PartitionFilters: [], PushedFilters: [IsNotNull(Code_DA2)], ReadSchema: struct<Code_DA2:string>
Project [Code_DA3#2648 AS COL_983A817F_3EA2_4B94_AD4A_A0B9FE6C2965#3081]
Filter isnotnull(Code_DA3#2648)
WholeStageCodegen (3)
FileScan csv [Code_DA3#2648] Batched: false, DataFilters: [isnotnull(Code_DA3#2648)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/execution/DIAGNOSTIC_14331/current/diagnost..., PartitionFilters: [], PushedFilters: [IsNotNull(Code_DA3)], ReadSchema: struct<Code_DA3:string>
Project [Code_DA4#2693 AS COL_983A817F_3EA2_4B94_AD4A_A0B9FE6C2965#3089]
Filter isnotnull(Code_DA4#2693)
WholeStageCodegen (4)
FileScan csv [Code_DA4#2693] Batched: false, DataFilters: [isnotnull(Code_DA4#2693)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/execution/DIAGNOSTIC_14331/current/diagnost..., PartitionFilters: [], PushedFilters: [IsNotNull(Code_DA4)], ReadSchema: struct<Code_DA4:string>
Project [Code_DA5#2738 AS COL_983A817F_3EA2_4B94_AD4A_A0B9FE6C2965#3097]
Filter isnotnull(Code_DA5#2738)
WholeStageCodegen (5)
FileScan csv [Code_DA5#2738] Batched: false, DataFilters: [isnotnull(Code_DA5#2738)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/execution/DIAGNOSTIC_14331/current/diagnost..., PartitionFilters: [], PushedFilters: [IsNotNull(Code_DA5)], ReadSchema: struct<Code_DA5:string>
Sort [REGROUPEMENT_CO_14459_CODE_CIM10_14460#2791 ASC NULLS FIRST], false, 0
WholeStageCodegen (8)
AQEShuffleRead coalesced
Exchange hashpartitioning(REGROUPEMENT_CO_14459_CODE_CIM10_14460#2791, 10), ENSURE_REQUIREMENTS, [plan_id=1154]
Project [Code_CIM10#2187 AS REGROUPEMENT_CO_14459_CODE_CIM10_14460#2791]
Filter (isnotnull(CCS_Famille#2190) AND isnotnull(Code_CIM10#2187))
WholeStageCodegen (6)
FileScan csv [Code_CIM10#2187,CCS_Famille#2190] Batched: false, DataFilters: [isnotnull(CCS_Famille#2190), isnotnull(Code_CIM10#2187)], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/data/input/depot/csv/execution/REGROUPEMENT_CO_14336/current/ccs..., PartitionFilters: [], PushedFilters: [IsNotNull(CCS_Famille), IsNotNull(Code_CIM10)], ReadSchema: struct<Code_CIM10:string,CCS_Famille:string>