digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: 0 ms";
2 [labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 0 ms<br>number of output rows: 1"];
}
3 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 1<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 1<br>local bytes read: 59.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 16.0 B<br>local merged bytes read: 0.0 B<br>number of partitions: 1<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 59.0 B"];
subgraph cluster4 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: 1 ms";
5 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 0 ms<br>peak memory: 0.0 B<br>number of output rows: 1<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
6 [labelType="html" label="<br><b>Project</b><br><br>"];
7 [labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 34<br>spill size: 0.0 B"];
}
subgraph cluster8 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: 3 ms";
9 [labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 64.1 MiB<br>spill size: 0.0 B"];
}
10 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 2.2 KiB<br>number of coalesced partitions: 1"];
11 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 34<br>local merged chunks fetched: 0<br>shuffle write time: 2 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 34<br>local bytes read: 2043.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 1904.0 B<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 2043.0 B"];
subgraph cluster12 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: 18 ms";
13 [labelType="html" label="<br><b>Project</b><br><br>"];
14 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 34"];
15 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 1,382<br>number of input batches: 1"];
}
16 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 1<br>scan time: 10 ms<br>metadata time: 0 ms<br>size of files read: 127.1 KiB<br>number of output rows: 1,382"];
subgraph cluster17 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: 0 ms";
18 [labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 64.1 MiB<br>spill size: 0.0 B"];
}
19 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 9.4 KiB<br>number of coalesced partitions: 1"];
20 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 189<br>local merged chunks fetched: 0<br>shuffle write time: 1 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 189<br>local bytes read: 8.8 KiB<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 10.3 KiB<br>local merged bytes read: 0.0 B<br>number of partitions: 10<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 8.8 KiB"];
subgraph cluster21 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: 97 ms";
22 [labelType="html" label="<br><b>Project</b><br><br>"];
23 [labelType="html" label="<b>Scan JDBCRelation((SELECT RAWTOHEX(PATIENT_ID), \"BIRTH_DATE\", \"GENDER\", \"PSEUDO\", \"INHOSPITAL_DEATH_DATE\", \"INHOSPITAL_DEATH_CODE\", \"DEATH_DATE\", \"EXTERNAL_DEATH_PROCESS_ID\" FROM \"DTM_622\".\"PATIENT\") x) [numPartitions=1] </b><br><br>number of output rows: 189"];
}
2->0;
3->2;
5->3;
6->5;
7->6;
9->7;
10->9;
11->10;
13->11;
14->13;
15->14;
16->15;
18->7;
19->18;
20->19;
22->20;
23->22;
}
24
AdaptiveSparkPlan isFinalPlan=true
HashAggregate(keys=[], functions=[count(1)])
WholeStageCodegen (6)
Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11901]
HashAggregate(keys=[], functions=[partial_count(1)])
Project
SortMergeJoin [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504], [PATIENT_3332_PATIENT_ID_3333#70880], LeftOuter
WholeStageCodegen (5)
Sort [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504 ASC NULLS FIRST], false, 0
WholeStageCodegen (3)
AQEShuffleRead coalesced
Exchange hashpartitioning(COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504, 10), ENSURE_REQUIREMENTS, [plan_id=11663]
Project [PATIENT_ID_3360#70398 AS COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504]
Filter (UNIT_3361#70399 <=> 2322)
ColumnarToRow
WholeStageCodegen (1)
FileScan parquet [PATIENT_ID_3360#70398,UNIT_3361#70399] Batched: true, DataFilters: [(UNIT_3361#70399 <=> 2322)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/data/output/cache/parquet/uet/VISIT_UNIT_3357], PartitionFilters: [], PushedFilters: [EqualNullSafe(UNIT_3361,2322)], ReadSchema: struct<PATIENT_ID_3360:string,UNIT_3361:string>
Sort [PATIENT_3332_PATIENT_ID_3333#70880 ASC NULLS FIRST], false, 0
WholeStageCodegen (4)
AQEShuffleRead coalesced
Exchange hashpartitioning(PATIENT_3332_PATIENT_ID_3333#70880, 10), ENSURE_REQUIREMENTS, [plan_id=11691]
Project [RAWTOHEX(PATIENT_ID)#70194 AS PATIENT_3332_PATIENT_ID_3333#70880]
Scan JDBCRelation((SELECT RAWTOHEX(PATIENT_ID), "BIRTH_DATE", "GENDER", "PSEUDO", "INHOSPITAL_DEATH_DATE", "INHOSPITAL_DEATH_CODE", "DEATH_DATE", "EXTERNAL_DEATH_PROCESS_ID" FROM "DTM_622"."PATIENT") x) [numPartitions=1] [RAWTOHEX(PATIENT_ID)#70194] PushedFilters: [*IsNotNull(RAWTOHEX(PATIENT_ID))], ReadSchema: struct<RAWTOHEX(PATIENT_ID):string>
WholeStageCodegen (2)
== Physical Plan ==
AdaptiveSparkPlan (32)
+- == Final Plan ==
* HashAggregate (20)
+- ShuffleQueryStage (19), Statistics(sizeInBytes=16.0 B, rowCount=1)
+- Exchange (18)
+- * HashAggregate (17)
+- * Project (16)
+- * SortMergeJoin LeftOuter (15)
:- * Sort (8)
: +- AQEShuffleRead (7)
: +- ShuffleQueryStage (6), Statistics(sizeInBytes=1904.0 B, rowCount=34)
: +- Exchange (5)
: +- * Project (4)
: +- * Filter (3)
: +- * ColumnarToRow (2)
: +- Scan parquet (1)
+- * Sort (14)
+- AQEShuffleRead (13)
+- ShuffleQueryStage (12), Statistics(sizeInBytes=10.3 KiB, rowCount=189)
+- Exchange (11)
+- * Project (10)
+- * Scan JDBCRelation((SELECT RAWTOHEX(PATIENT_ID), "BIRTH_DATE", "GENDER", "PSEUDO", "INHOSPITAL_DEATH_DATE", "INHOSPITAL_DEATH_CODE", "DEATH_DATE", "EXTERNAL_DEATH_PROCESS_ID" FROM "DTM_622"."PATIENT") x) [numPartitions=1] (9)
+- == Initial Plan ==
HashAggregate (31)
+- Exchange (30)
+- HashAggregate (29)
+- Project (28)
+- SortMergeJoin LeftOuter (27)
:- Sort (24)
: +- Exchange (23)
: +- Project (22)
: +- Filter (21)
: +- Scan parquet (1)
+- Sort (26)
+- Exchange (25)
+- Project (10)
+- Scan JDBCRelation((SELECT RAWTOHEX(PATIENT_ID), "BIRTH_DATE", "GENDER", "PSEUDO", "INHOSPITAL_DEATH_DATE", "INHOSPITAL_DEATH_CODE", "DEATH_DATE", "EXTERNAL_DEATH_PROCESS_ID" FROM "DTM_622"."PATIENT") x) [numPartitions=1] (9)
(1) Scan parquet
Output [2]: [PATIENT_ID_3360#70398, UNIT_3361#70399]
Batched: true
Location: InMemoryFileIndex [file:/data/output/cache/parquet/uet/VISIT_UNIT_3357]
PushedFilters: [EqualNullSafe(UNIT_3361,2322)]
ReadSchema: struct<PATIENT_ID_3360:string,UNIT_3361:string>
(2) ColumnarToRow [codegen id : 1]
Input [2]: [PATIENT_ID_3360#70398, UNIT_3361#70399]
(3) Filter [codegen id : 1]
Input [2]: [PATIENT_ID_3360#70398, UNIT_3361#70399]
Condition : (UNIT_3361#70399 <=> 2322)
(4) Project [codegen id : 1]
Output [1]: [PATIENT_ID_3360#70398 AS COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504]
Input [2]: [PATIENT_ID_3360#70398, UNIT_3361#70399]
(5) Exchange
Input [1]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504]
Arguments: hashpartitioning(COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504, 10), ENSURE_REQUIREMENTS, [plan_id=11663]
(6) ShuffleQueryStage
Output [1]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504]
Arguments: 0
(7) AQEShuffleRead
Input [1]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504]
Arguments: coalesced
(8) Sort [codegen id : 3]
Input [1]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504]
Arguments: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504 ASC NULLS FIRST], false, 0
(9) Scan JDBCRelation((SELECT RAWTOHEX(PATIENT_ID), "BIRTH_DATE", "GENDER", "PSEUDO", "INHOSPITAL_DEATH_DATE", "INHOSPITAL_DEATH_CODE", "DEATH_DATE", "EXTERNAL_DEATH_PROCESS_ID" FROM "DTM_622"."PATIENT") x) [numPartitions=1] [codegen id : 2]
Output [1]: [RAWTOHEX(PATIENT_ID)#70194]
PushedFilters: [*IsNotNull(RAWTOHEX(PATIENT_ID))]
ReadSchema: struct<RAWTOHEX(PATIENT_ID):string>
(10) Project [codegen id : 2]
Output [1]: [RAWTOHEX(PATIENT_ID)#70194 AS PATIENT_3332_PATIENT_ID_3333#70880]
Input [1]: [RAWTOHEX(PATIENT_ID)#70194]
(11) Exchange
Input [1]: [PATIENT_3332_PATIENT_ID_3333#70880]
Arguments: hashpartitioning(PATIENT_3332_PATIENT_ID_3333#70880, 10), ENSURE_REQUIREMENTS, [plan_id=11691]
(12) ShuffleQueryStage
Output [1]: [PATIENT_3332_PATIENT_ID_3333#70880]
Arguments: 1
(13) AQEShuffleRead
Input [1]: [PATIENT_3332_PATIENT_ID_3333#70880]
Arguments: coalesced
(14) Sort [codegen id : 4]
Input [1]: [PATIENT_3332_PATIENT_ID_3333#70880]
Arguments: [PATIENT_3332_PATIENT_ID_3333#70880 ASC NULLS FIRST], false, 0
(15) SortMergeJoin [codegen id : 5]
Left keys [1]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504]
Right keys [1]: [PATIENT_3332_PATIENT_ID_3333#70880]
Join type: LeftOuter
Join condition: None
(16) Project [codegen id : 5]
Output: []
Input [2]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504, PATIENT_3332_PATIENT_ID_3333#70880]
(17) HashAggregate [codegen id : 5]
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#71683L]
Results [1]: [count#71684L]
(18) Exchange
Input [1]: [count#71684L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11901]
(19) ShuffleQueryStage
Output [1]: [count#71684L]
Arguments: 2
(20) HashAggregate [codegen id : 6]
Input [1]: [count#71684L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#71680L]
Results [1]: [count(1)#71680L AS count#71681L]
(21) Filter
Input [2]: [PATIENT_ID_3360#70398, UNIT_3361#70399]
Condition : (UNIT_3361#70399 <=> 2322)
(22) Project
Output [1]: [PATIENT_ID_3360#70398 AS COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504]
Input [2]: [PATIENT_ID_3360#70398, UNIT_3361#70399]
(23) Exchange
Input [1]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504]
Arguments: hashpartitioning(COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504, 10), ENSURE_REQUIREMENTS, [plan_id=11603]
(24) Sort
Input [1]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504]
Arguments: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504 ASC NULLS FIRST], false, 0
(25) Exchange
Input [1]: [PATIENT_3332_PATIENT_ID_3333#70880]
Arguments: hashpartitioning(PATIENT_3332_PATIENT_ID_3333#70880, 10), ENSURE_REQUIREMENTS, [plan_id=11604]
(26) Sort
Input [1]: [PATIENT_3332_PATIENT_ID_3333#70880]
Arguments: [PATIENT_3332_PATIENT_ID_3333#70880 ASC NULLS FIRST], false, 0
(27) SortMergeJoin
Left keys [1]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504]
Right keys [1]: [PATIENT_3332_PATIENT_ID_3333#70880]
Join type: LeftOuter
Join condition: None
(28) Project
Output: []
Input [2]: [COL_FC9902B4_8859_4AF9_ADF1_99A2E15E5351#70504, PATIENT_3332_PATIENT_ID_3333#70880]
(29) HashAggregate
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#71683L]
Results [1]: [count#71684L]
(30) Exchange
Input [1]: [count#71684L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11611]
(31) HashAggregate
Input [1]: [count#71684L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#71680L]
Results [1]: [count(1)#71680L AS count#71681L]
(32) AdaptiveSparkPlan
Output [1]: [count#71681L]
Arguments: isFinalPlan=true