digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
1 [labelType="html" label="<br><b>TakeOrderedAndProject</b><br><br>"];
subgraph cluster2 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: 16 ms";
3 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 0 ms<br>peak memory: 64.2 MiB<br>number of output rows: 81<br>number of sort fallback tasks: 0<br>avg hash probes per key: 1"];
}
4 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 1606.0 B<br>number of coalesced partitions: 1"];
5 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 81<br>local merged chunks fetched: 0<br>shuffle write time: 2 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 81<br>local bytes read: 1527.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 2.5 KiB<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 1527.0 B"];
subgraph cluster6 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: 71 ms";
7 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 52 ms<br>peak memory: 256.0 KiB<br>number of output rows: 81<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
8 [labelType="html" label="<br><b>Project</b><br><br>"];
9 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 1,382<br>spill size: 0.0 B"];
}
subgraph cluster10 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: 98 ms";
11 [labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 64.1 MiB<br>spill size: 0.0 B"];
}
12 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 20.0 KiB<br>number of coalesced partitions: 1"];
13 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 1,382<br>local merged chunks fetched: 0<br>shuffle write time: 2 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 1,382<br>local bytes read: 18.8 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 97.2 KiB<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 18.8 KiB"];
subgraph cluster14 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: 10 ms";
15 [labelType="html" label="<br><b>Project</b><br><br>"];
16 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 1,382"];
17 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 1,382<br>number of input batches: 1"];
}
18 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 1<br>scan time: 7 ms<br>metadata time: 0 ms<br>size of files read: 127.1 KiB<br>number of output rows: 1,382"];
subgraph cluster19 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: 97 ms";
20 [labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 64.1 MiB<br>spill size: 0.0 B"];
}
21 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 9.4 KiB<br>number of coalesced partitions: 1"];
22 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 189<br>local merged chunks fetched: 0<br>shuffle write time: 2 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 189<br>local bytes read: 8.8 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 10.3 KiB<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 8.8 KiB"];
subgraph cluster23 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: 99 ms";
24 [labelType="html" label="<br><b>Project</b><br><br>"];
25 [labelType="html" label="<b>Scan JDBCRelation((SELECT RAWTOHEX(PATIENT_ID), \"BIRTH_DATE\", \"GENDER\", \"PSEUDO\", \"INHOSPITAL_DEATH_DATE\", \"INHOSPITAL_DEATH_CODE\", \"DEATH_DATE\", \"EXTERNAL_DEATH_PROCESS_ID\" FROM \"DTM_622\".\"PATIENT\") x) [numPartitions=1] </b><br><br>number of output rows: 189"];
}
1->0;
3->1;
4->3;
5->4;
7->5;
8->7;
9->8;
11->9;
12->11;
13->12;
15->13;
16->15;
17->16;
18->17;
20->9;
21->20;
22->21;
24->22;
25->24;
}
26
AdaptiveSparkPlan isFinalPlan=true
TakeOrderedAndProject(limit=10, orderBy=[COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE2#68574L DESC NULLS LAST], output=[COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE1#68553,COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE2#68574L])
HashAggregate(keys=[COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260], functions=[count(1)])
WholeStageCodegen (6)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260, 10), ENSURE_REQUIREMENTS, [plan_id=11127]
HashAggregate(keys=[COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260], functions=[partial_count(1)])
Project [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
SortMergeJoin [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259], [PATIENT_3332_PATIENT_ID_3333#68443], LeftOuter
WholeStageCodegen (5)
Sort [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259 ASC NULLS FIRST], false, 0
WholeStageCodegen (3)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259, 10), ENSURE_REQUIREMENTS, [plan_id=11018]
Project [PATIENT_ID_3360#68204 AS COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259, UNIT_3361#68205 AS COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Filter isnotnull(UNIT_3361#68205)
ColumnarToRow
WholeStageCodegen (1)
FileScan parquet [PATIENT_ID_3360#68204,UNIT_3361#68205] Batched: true, DataFilters: [isnotnull(UNIT_3361#68205)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/uet/VISIT_UNIT_3357], PartitionFilters: [], PushedFilters: [IsNotNull(UNIT_3361)], ReadSchema: struct<PATIENT_ID_3360:string,UNIT_3361:string>
Sort [PATIENT_3332_PATIENT_ID_3333#68443 ASC NULLS FIRST], false, 0
WholeStageCodegen (4)
AQEShuffleRead coalesced
Exchange hashpartitioning(PATIENT_3332_PATIENT_ID_3333#68443, 10), ENSURE_REQUIREMENTS, [plan_id=11031]
Project [RAWTOHEX(PATIENT_ID)#68121 AS PATIENT_3332_PATIENT_ID_3333#68443]
Scan JDBCRelation((SELECT RAWTOHEX(PATIENT_ID), "BIRTH_DATE", "GENDER", "PSEUDO", "INHOSPITAL_DEATH_DATE", "INHOSPITAL_DEATH_CODE", "DEATH_DATE", "EXTERNAL_DEATH_PROCESS_ID" FROM "DTM_622"."PATIENT") x) [numPartitions=1] [RAWTOHEX(PATIENT_ID)#68121] PushedFilters: [*IsNotNull(RAWTOHEX(PATIENT_ID))], ReadSchema: struct<RAWTOHEX(PATIENT_ID):string>
WholeStageCodegen (2)
== Physical Plan ==
AdaptiveSparkPlan (35)
+- == Final Plan ==
TakeOrderedAndProject (22)
+- * HashAggregate (21)
+- AQEShuffleRead (20)
+- ShuffleQueryStage (19), Statistics(sizeInBytes=2.5 KiB, rowCount=81)
+- Exchange (18)
+- * HashAggregate (17)
+- * Project (16)
+- * SortMergeJoin LeftOuter (15)
:- * Sort (8)
: +- AQEShuffleRead (7)
: +- ShuffleQueryStage (6), Statistics(sizeInBytes=97.2 KiB, rowCount=1.38E+3)
: +- Exchange (5)
: +- * Project (4)
: +- * Filter (3)
: +- * ColumnarToRow (2)
: +- Scan parquet (1)
+- * Sort (14)
+- AQEShuffleRead (13)
+- ShuffleQueryStage (12), Statistics(sizeInBytes=10.3 KiB, rowCount=189)
+- Exchange (11)
+- * Project (10)
+- * Scan JDBCRelation((SELECT RAWTOHEX(PATIENT_ID), "BIRTH_DATE", "GENDER", "PSEUDO", "INHOSPITAL_DEATH_DATE", "INHOSPITAL_DEATH_CODE", "DEATH_DATE", "EXTERNAL_DEATH_PROCESS_ID" FROM "DTM_622"."PATIENT") x) [numPartitions=1] (9)
+- == Initial Plan ==
TakeOrderedAndProject (34)
+- HashAggregate (33)
+- Exchange (32)
+- HashAggregate (31)
+- Project (30)
+- SortMergeJoin LeftOuter (29)
:- Sort (26)
: +- Exchange (25)
: +- Project (24)
: +- Filter (23)
: +- Scan parquet (1)
+- Sort (28)
+- Exchange (27)
+- Project (10)
+- Scan JDBCRelation((SELECT RAWTOHEX(PATIENT_ID), "BIRTH_DATE", "GENDER", "PSEUDO", "INHOSPITAL_DEATH_DATE", "INHOSPITAL_DEATH_CODE", "DEATH_DATE", "EXTERNAL_DEATH_PROCESS_ID" FROM "DTM_622"."PATIENT") x) [numPartitions=1] (9)
(1) Scan parquet
Output [2]: [PATIENT_ID_3360#68204, UNIT_3361#68205]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/uet/VISIT_UNIT_3357]
PushedFilters: [IsNotNull(UNIT_3361)]
ReadSchema: struct<PATIENT_ID_3360:string,UNIT_3361:string>
(2) ColumnarToRow [codegen id : 1]
Input [2]: [PATIENT_ID_3360#68204, UNIT_3361#68205]
(3) Filter [codegen id : 1]
Input [2]: [PATIENT_ID_3360#68204, UNIT_3361#68205]
Condition : isnotnull(UNIT_3361#68205)
(4) Project [codegen id : 1]
Output [2]: [PATIENT_ID_3360#68204 AS COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259, UNIT_3361#68205 AS COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Input [2]: [PATIENT_ID_3360#68204, UNIT_3361#68205]
(5) Exchange
Input [2]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259, COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Arguments: hashpartitioning(COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259, 10), ENSURE_REQUIREMENTS, [plan_id=11018]
(6) ShuffleQueryStage
Output [2]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259, COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Arguments: 0
(7) AQEShuffleRead
Input [2]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259, COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Arguments: coalesced
(8) Sort [codegen id : 3]
Input [2]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259, COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Arguments: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259 ASC NULLS FIRST], false, 0
(9) Scan JDBCRelation((SELECT RAWTOHEX(PATIENT_ID), "BIRTH_DATE", "GENDER", "PSEUDO", "INHOSPITAL_DEATH_DATE", "INHOSPITAL_DEATH_CODE", "DEATH_DATE", "EXTERNAL_DEATH_PROCESS_ID" FROM "DTM_622"."PATIENT") x) [numPartitions=1] [codegen id : 2]
Output [1]: [RAWTOHEX(PATIENT_ID)#68121]
PushedFilters: [*IsNotNull(RAWTOHEX(PATIENT_ID))]
ReadSchema: struct<RAWTOHEX(PATIENT_ID):string>
(10) Project [codegen id : 2]
Output [1]: [RAWTOHEX(PATIENT_ID)#68121 AS PATIENT_3332_PATIENT_ID_3333#68443]
Input [1]: [RAWTOHEX(PATIENT_ID)#68121]
(11) Exchange
Input [1]: [PATIENT_3332_PATIENT_ID_3333#68443]
Arguments: hashpartitioning(PATIENT_3332_PATIENT_ID_3333#68443, 10), ENSURE_REQUIREMENTS, [plan_id=11031]
(12) ShuffleQueryStage
Output [1]: [PATIENT_3332_PATIENT_ID_3333#68443]
Arguments: 1
(13) AQEShuffleRead
Input [1]: [PATIENT_3332_PATIENT_ID_3333#68443]
Arguments: coalesced
(14) Sort [codegen id : 4]
Input [1]: [PATIENT_3332_PATIENT_ID_3333#68443]
Arguments: [PATIENT_3332_PATIENT_ID_3333#68443 ASC NULLS FIRST], false, 0
(15) SortMergeJoin [codegen id : 5]
Left keys [1]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259]
Right keys [1]: [PATIENT_3332_PATIENT_ID_3333#68443]
Join type: LeftOuter
Join condition: None
(16) Project [codegen id : 5]
Output [1]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Input [3]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259, COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260, PATIENT_3332_PATIENT_ID_3333#68443]
(17) HashAggregate [codegen id : 5]
Input [1]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Keys [1]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#68581L]
Results [2]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260, count#68582L]
(18) Exchange
Input [2]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260, count#68582L]
Arguments: hashpartitioning(COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260, 10), ENSURE_REQUIREMENTS, [plan_id=11127]
(19) ShuffleQueryStage
Output [2]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260, count#68582L]
Arguments: 2
(20) AQEShuffleRead
Input [2]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260, count#68582L]
Arguments: coalesced
(21) HashAggregate [codegen id : 6]
Input [2]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260, count#68582L]
Keys [1]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#68572L]
Results [2]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260 AS COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE1#68553, count(1)#68572L AS COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE2#68574L]
(22) TakeOrderedAndProject
Input [2]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE1#68553, COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE2#68574L]
Arguments: 10, [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE2#68574L DESC NULLS LAST], [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE1#68553, COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE2#68574L]
(23) Filter
Input [2]: [PATIENT_ID_3360#68204, UNIT_3361#68205]
Condition : isnotnull(UNIT_3361#68205)
(24) Project
Output [2]: [PATIENT_ID_3360#68204 AS COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259, UNIT_3361#68205 AS COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Input [2]: [PATIENT_ID_3360#68204, UNIT_3361#68205]
(25) Exchange
Input [2]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259, COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Arguments: hashpartitioning(COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259, 10), ENSURE_REQUIREMENTS, [plan_id=10974]
(26) Sort
Input [2]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259, COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Arguments: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259 ASC NULLS FIRST], false, 0
(27) Exchange
Input [1]: [PATIENT_3332_PATIENT_ID_3333#68443]
Arguments: hashpartitioning(PATIENT_3332_PATIENT_ID_3333#68443, 10), ENSURE_REQUIREMENTS, [plan_id=10975]
(28) Sort
Input [1]: [PATIENT_3332_PATIENT_ID_3333#68443]
Arguments: [PATIENT_3332_PATIENT_ID_3333#68443 ASC NULLS FIRST], false, 0
(29) SortMergeJoin
Left keys [1]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259]
Right keys [1]: [PATIENT_3332_PATIENT_ID_3333#68443]
Join type: LeftOuter
Join condition: None
(30) Project
Output [1]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Input [3]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#68259, COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260, PATIENT_3332_PATIENT_ID_3333#68443]
(31) HashAggregate
Input [1]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Keys [1]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#68581L]
Results [2]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260, count#68582L]
(32) Exchange
Input [2]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260, count#68582L]
Arguments: hashpartitioning(COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260, 10), ENSURE_REQUIREMENTS, [plan_id=10982]
(33) HashAggregate
Input [2]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260, count#68582L]
Keys [1]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260]
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#68572L]
Results [2]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE#68260 AS COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE1#68553, count(1)#68572L AS COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE2#68574L]
(34) TakeOrderedAndProject
Input [2]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE1#68553, COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE2#68574L]
Arguments: 10, [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE2#68574L DESC NULLS LAST], [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE1#68553, COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE2#68574L]
(35) AdaptiveSparkPlan
Output [2]: [COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE1#68553, COL_9A5ACA40_F512_495A_9718_534E3BBBA5EE2#68574L]
Arguments: isFinalPlan=true