Skip to content

Commit 1b9baad

Browse files
author
Andreas Neumann
authored
Merge pull request #23 from data-integrations/feature/v2.2.0-rename-and-spark2
v2.2.0 to rename packages, rebase on 6.0.0, update to spark 2.3
2 parents eb1ad1b + b2be8fa commit 1b9baad

File tree

11 files changed

+190
-160
lines changed

11 files changed

+190
-160
lines changed

‎.travis.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright © 2017 Cask Data, Inc.
2+
# Copyright © 2017-2019 Cask Data, Inc.
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
55
# use this file except in compliance with the License. You may obtain a copy of
@@ -19,7 +19,6 @@ dist: precise
1919
language: java
2020

2121
jdk:
22-
- oraclejdk7
2322
- oraclejdk8
2423

2524
branches:

‎pom.xml‎

Lines changed: 44 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<?xml version="1.0" encoding="UTF-8"?>
22
<!--
3-
~ Copyright © 2017 Cask Data, Inc.
3+
~ Copyright © 2017-2019 Cask Data, Inc.
44
~
55
~ Licensed under the Apache License, Version 2.0 (the "License"); you may not
66
~ use this file except in compliance with the License. You may obtain a copy of
@@ -20,21 +20,17 @@
2020
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
2121
<modelVersion>4.0.0</modelVersion>
2222

23-
<groupId>co.cask.hydrator</groupId>
23+
<groupId>io.cdap.plugin</groupId>
2424
<artifactId>dynamic-spark</artifactId>
25-
<version>2.2.0-SNAPSHOT</version>
25+
<version>2.2.0</version>
2626

2727
<properties>
2828
<!-- properties for script build step that creates the config files for the artifacts -->
29-
<widgets.dir>widgets</widgets.dir>
30-
<docs.dir>docs</docs.dir>
31-
<data.pipeline.parent>system:cdap-data-pipeline[4.3.0-SNAPSHOT,6.0.0-SNAPSHOT)</data.pipeline.parent>
32-
<data.stream.parent>system:cdap-data-streams[4.3.0-SNAPSHOT,6.0.0-SNAPSHOT)</data.stream.parent>
33-
<!-- this is here because project.basedir evaluates to null in the script build step -->
34-
<main.basedir>${project.basedir}</main.basedir>
29+
<data.pipeline.parent>system:cdap-data-pipeline[6.0.0-SNAPSHOT,7.0.0-SNAPSHOT)</data.pipeline.parent>
30+
<data.stream.parent>system:cdap-data-streams[6.0.0-SNAPSHOT,7.0.0-SNAPSHOT)</data.stream.parent>
3531

36-
<cdap.version>4.3.0</cdap.version>
37-
<spark.version>1.6.3</spark.version>
32+
<cdap.version>6.0.0-SNAPSHOT</cdap.version>
33+
<spark.version>2.1.3</spark.version>
3834
<logback.version>1.0.9</logback.version>
3935

4036
<surefire.redirectTestOutputToFile>true</surefire.redirectTestOutputToFile>
@@ -49,14 +45,14 @@
4945

5046
<dependencies>
5147
<dependency>
52-
<groupId>co.cask.cdap</groupId>
48+
<groupId>io.cdap.cdap</groupId>
5349
<artifactId>cdap-api</artifactId>
5450
<version>${cdap.version}</version>
5551
<scope>provided</scope>
5652
</dependency>
5753
<dependency>
58-
<groupId>co.cask.cdap</groupId>
59-
<artifactId>cdap-api-spark</artifactId>
54+
<groupId>io.cdap.cdap</groupId>
55+
<artifactId>cdap-api-spark2_2.11</artifactId>
6056
<version>${cdap.version}</version>
6157
<scope>provided</scope>
6258
<exclusions>
@@ -67,20 +63,20 @@
6763
</exclusions>
6864
</dependency>
6965
<dependency>
70-
<groupId>co.cask.cdap</groupId>
66+
<groupId>io.cdap.cdap</groupId>
7167
<artifactId>cdap-etl-api</artifactId>
7268
<version>${cdap.version}</version>
7369
<scope>provided</scope>
7470
</dependency>
7571
<dependency>
76-
<groupId>co.cask.cdap</groupId>
72+
<groupId>io.cdap.cdap</groupId>
7773
<artifactId>cdap-etl-api-spark</artifactId>
7874
<version>${cdap.version}</version>
7975
<scope>provided</scope>
8076
</dependency>
8177
<dependency>
8278
<groupId>org.apache.spark</groupId>
83-
<artifactId>spark-repl_2.10</artifactId>
79+
<artifactId>spark-repl_2.11</artifactId>
8480
<version>${spark.version}</version>
8581
<scope>provided</scope>
8682
<exclusions>
@@ -92,6 +88,10 @@
9288
<groupId>org.slf4j</groupId>
9389
<artifactId>slf4j-log4j12</artifactId>
9490
</exclusion>
91+
<exclusion>
92+
<groupId>io.netty</groupId>
93+
<artifactId>*</artifactId>
94+
</exclusion>
9595
</exclusions>
9696
</dependency>
9797
<dependency>
@@ -101,20 +101,36 @@
101101
<scope>test</scope>
102102
</dependency>
103103
<dependency>
104-
<groupId>co.cask.cdap</groupId>
104+
<groupId>io.cdap.cdap</groupId>
105105
<artifactId>hydrator-test</artifactId>
106106
<version>${cdap.version}</version>
107107
<scope>test</scope>
108108
<exclusions>
109109
<exclusion>
110-
<groupId>co.cask.cdap</groupId>
110+
<groupId>io.cdap.cdap</groupId>
111+
<artifactId>cdap-unit-test</artifactId>
112+
</exclusion>
113+
</exclusions>
114+
</dependency>
115+
<dependency>
116+
<groupId>io.cdap.cdap</groupId>
117+
<artifactId>cdap-unit-test-spark2_2.11</artifactId>
118+
<version>${cdap.version}</version>
119+
<scope>test</scope>
120+
<exclusions>
121+
<exclusion>
122+
<groupId>asm</groupId>
123+
<artifactId>asm</artifactId>
124+
</exclusion>
125+
<exclusion>
126+
<groupId>io.cdap.cdap</groupId>
111127
<artifactId>cdap-explore-jdbc</artifactId>
112128
</exclusion>
113129
</exclusions>
114130
</dependency>
115131
<dependency>
116-
<groupId>co.cask.cdap</groupId>
117-
<artifactId>cdap-data-pipeline</artifactId>
132+
<groupId>io.cdap.cdap</groupId>
133+
<artifactId>cdap-data-pipeline2_2.11</artifactId>
118134
<version>${cdap.version}</version>
119135
<scope>test</scope>
120136
</dependency>
@@ -206,16 +222,16 @@
206222
<artifactId>maven-compiler-plugin</artifactId>
207223
<version>3.1</version>
208224
<configuration>
209-
<source>1.7</source>
210-
<target>1.7</target>
225+
<source>1.8</source>
226+
<target>1.8</target>
211227
</configuration>
212228
</plugin>
213229
<plugin>
214230
<groupId>org.apache.maven.plugins</groupId>
215231
<artifactId>maven-surefire-plugin</artifactId>
216232
<version>2.14.1</version>
217233
<configuration>
218-
<argLine>-Xmx5000m -Djava.awt.headless=true -XX:MaxPermSize=256m -XX:+UseConcMarkSweepGC
234+
<argLine>-Xmx5000m -Djava.awt.headless=true -XX:+UseConcMarkSweepGC
219235
-XX:OnOutOfMemoryError="kill -9 %p" -XX:+HeapDumpOnOutOfMemoryError
220236
</argLine>
221237
<redirectTestOutputToFile>${surefire.redirectTestOutputToFile}</redirectTestOutputToFile>
@@ -250,7 +266,7 @@
250266
<configuration>
251267
<instructions>
252268
<_exportcontents>
253-
co.cask.hydrator.plugin.spark.*;
269+
io.cdap.plugin.spark.*;
254270
</_exportcontents>
255271
<Embed-Dependency>*;inline=false;scope=compile</Embed-Dependency>
256272
<Embed-Transitive>true</Embed-Transitive>
@@ -267,9 +283,9 @@
267283
</executions>
268284
</plugin>
269285
<plugin>
270-
<groupId>co.cask</groupId>
286+
<groupId>io.cdap</groupId>
271287
<artifactId>cdap-maven-plugin</artifactId>
272-
<version>1.0.1-SNAPSHOT</version>
288+
<version>1.1.0</version>
273289
<configuration>
274290
<cdapArtifacts>
275291
<parent>${data.pipeline.parent}</parent>
@@ -303,7 +319,6 @@
303319
<configuration>
304320
<nexusUrl>https://oss.sonatype.org</nexusUrl>
305321
<serverId>sonatype.release</serverId>
306-
<stagingProfileId>655dc88dc770c3</stagingProfileId>
307322
</configuration>
308323
</plugin>
309324
<!-- Source JAR -->
@@ -336,7 +351,7 @@
336351
</links>
337352
<doctitle>${project.name} ${project.version}</doctitle>
338353
<bottom>
339-
<![CDATA[Copyright &#169; {currentYear} <a href="http://cask.co" target="_blank">Cask Data, Inc.</a> Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0" target="_blank">Apache License, Version 2.0</a>.]]>
354+
<![CDATA[Copyright &#169; {currentYear} <a href="http://cdap.io" target="_blank">CDAP</a> Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0" target="_blank">Apache License, Version 2.0</a>.]]>
340355
</bottom>
341356
</configuration>
342357
<executions>

src/main/java/co/cask/hydrator/plugin/spark/dynamic/ClassLoaders.java renamed to src/main/java/io/cdap/plugin/spark/dynamic/ClassLoaders.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright © 2017 Cask Data, Inc.
2+
* Copyright © 2017-2019 Cask Data, Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
55
* use this file except in compliance with the License. You may obtain a copy of
@@ -14,7 +14,7 @@
1414
* the License.
1515
*/
1616

17-
package co.cask.hydrator.plugin.spark.dynamic;
17+
package io.cdap.plugin.spark.dynamic;
1818

1919
import java.io.File;
2020
import java.net.URI;

src/main/java/co/cask/hydrator/plugin/spark/dynamic/PySparkProgram.java renamed to src/main/java/io/cdap/plugin/spark/dynamic/PySparkProgram.java

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright © 2017 Cask Data, Inc.
2+
* Copyright © 2017-2019 Cask Data, Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
55
* use this file except in compliance with the License. You may obtain a copy of
@@ -14,15 +14,15 @@
1414
* the License.
1515
*/
1616

17-
package co.cask.hydrator.plugin.spark.dynamic;
17+
package io.cdap.plugin.spark.dynamic;
1818

19-
import co.cask.cdap.api.annotation.Description;
20-
import co.cask.cdap.api.annotation.Macro;
21-
import co.cask.cdap.api.annotation.Name;
22-
import co.cask.cdap.api.annotation.Plugin;
23-
import co.cask.cdap.api.plugin.PluginConfig;
24-
import co.cask.cdap.api.spark.AbstractSpark;
25-
import co.cask.cdap.api.spark.Spark;
19+
import io.cdap.cdap.api.annotation.Description;
20+
import io.cdap.cdap.api.annotation.Macro;
21+
import io.cdap.cdap.api.annotation.Name;
22+
import io.cdap.cdap.api.annotation.Plugin;
23+
import io.cdap.cdap.api.plugin.PluginConfig;
24+
import io.cdap.cdap.api.spark.AbstractSpark;
25+
import io.cdap.cdap.api.spark.Spark;
2626

2727
import java.net.URI;
2828
import java.util.ArrayList;

src/main/java/co/cask/hydrator/plugin/spark/dynamic/RecordToRow.java renamed to src/main/java/io/cdap/plugin/spark/dynamic/RecordToRow.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright © 2018 Cask Data, Inc.
2+
* Copyright © 2018-2019 Cask Data, Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
55
* use this file except in compliance with the License. You may obtain a copy of
@@ -14,10 +14,10 @@
1414
* the License.
1515
*/
1616

17-
package co.cask.hydrator.plugin.spark.dynamic;
17+
package io.cdap.plugin.spark.dynamic;
1818

19-
import co.cask.cdap.api.data.format.StructuredRecord;
20-
import co.cask.cdap.api.spark.sql.DataFrames;
19+
import io.cdap.cdap.api.data.format.StructuredRecord;
20+
import io.cdap.cdap.api.spark.sql.DataFrames;
2121
import org.apache.spark.api.java.function.Function;
2222
import org.apache.spark.sql.Row;
2323
import org.apache.spark.sql.types.StructType;

src/main/java/co/cask/hydrator/plugin/spark/dynamic/ScalaSparkCodeExecutor.java renamed to src/main/java/io/cdap/plugin/spark/dynamic/ScalaSparkCodeExecutor.java

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright © 2018 Cask Data, Inc.
2+
* Copyright © 2018-2019 Cask Data, Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
55
* use this file except in compliance with the License. You may obtain a copy of
@@ -14,14 +14,14 @@
1414
* the License.
1515
*/
1616

17-
package co.cask.hydrator.plugin.spark.dynamic;
17+
package io.cdap.plugin.spark.dynamic;
1818

19-
import co.cask.cdap.api.data.format.StructuredRecord;
20-
import co.cask.cdap.api.data.schema.Schema;
21-
import co.cask.cdap.api.spark.dynamic.CompilationFailureException;
22-
import co.cask.cdap.api.spark.dynamic.SparkInterpreter;
23-
import co.cask.cdap.api.spark.sql.DataFrames;
24-
import co.cask.cdap.etl.api.batch.SparkExecutionPluginContext;
19+
import io.cdap.cdap.api.data.format.StructuredRecord;
20+
import io.cdap.cdap.api.data.schema.Schema;
21+
import io.cdap.cdap.api.spark.dynamic.CompilationFailureException;
22+
import io.cdap.cdap.api.spark.dynamic.SparkInterpreter;
23+
import io.cdap.cdap.api.spark.sql.DataFrames;
24+
import io.cdap.cdap.etl.api.batch.SparkExecutionPluginContext;
2525
import org.apache.spark.SparkContext;
2626
import org.apache.spark.SparkFirehoseListener;
2727
import org.apache.spark.api.java.JavaRDD;
@@ -51,7 +51,7 @@
5151
public class ScalaSparkCodeExecutor {
5252

5353
private static final Logger LOG = LoggerFactory.getLogger(ScalaSparkCodeExecutor.class);
54-
private static final String CLASS_NAME_PREFIX = "co.cask.hydrator.plugin.spark.dynamic.generated.UserScalaSpark$";
54+
private static final String CLASS_NAME_PREFIX = "io.cdap.plugin.spark.dynamic.generated.UserScalaSpark$";
5555
private static final Class<?> DATAFRAME_TYPE = getDataFrameType();
5656
private static final Class<?>[][] ACCEPTABLE_PARAMETER_TYPES = new Class<?>[][] {
5757
{ RDD.class, SparkExecutionPluginContext.class },
@@ -203,9 +203,9 @@ private String generateSourceClass(String className) {
203203
try (PrintWriter sourceWriter = new PrintWriter(writer, false)) {
204204
sourceWriter.println("package " + className.substring(0, className.lastIndexOf('.')));
205205
// Includes some commonly used imports.
206-
sourceWriter.println("import co.cask.cdap.api.data.format._");
207-
sourceWriter.println("import co.cask.cdap.api.data.schema._");
208-
sourceWriter.println("import co.cask.cdap.etl.api.batch._");
206+
sourceWriter.println("import io.cdap.cdap.api.data.format._");
207+
sourceWriter.println("import io.cdap.cdap.api.data.schema._");
208+
sourceWriter.println("import io.cdap.cdap.etl.api.batch._");
209209
sourceWriter.println("import org.apache.spark._");
210210
sourceWriter.println("import org.apache.spark.api.java._");
211211
sourceWriter.println("import org.apache.spark.rdd._");

src/main/java/co/cask/hydrator/plugin/spark/dynamic/ScalaSparkCompute.java renamed to src/main/java/io/cdap/plugin/spark/dynamic/ScalaSparkCompute.java

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright © 2017-2018 Cask Data, Inc.
2+
* Copyright © 2017-2019 Cask Data, Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
55
* use this file except in compliance with the License. You may obtain a copy of
@@ -14,20 +14,20 @@
1414
* the License.
1515
*/
1616

17-
package co.cask.hydrator.plugin.spark.dynamic;
18-
19-
import co.cask.cdap.api.annotation.Description;
20-
import co.cask.cdap.api.annotation.Macro;
21-
import co.cask.cdap.api.annotation.Name;
22-
import co.cask.cdap.api.annotation.Plugin;
23-
import co.cask.cdap.api.data.format.StructuredRecord;
24-
import co.cask.cdap.api.data.schema.Schema;
25-
import co.cask.cdap.api.plugin.PluginConfig;
26-
import co.cask.cdap.api.spark.sql.DataFrames;
27-
import co.cask.cdap.etl.api.PipelineConfigurer;
28-
import co.cask.cdap.etl.api.StageConfigurer;
29-
import co.cask.cdap.etl.api.batch.SparkCompute;
30-
import co.cask.cdap.etl.api.batch.SparkExecutionPluginContext;
17+
package io.cdap.plugin.spark.dynamic;
18+
19+
import io.cdap.cdap.api.annotation.Description;
20+
import io.cdap.cdap.api.annotation.Macro;
21+
import io.cdap.cdap.api.annotation.Name;
22+
import io.cdap.cdap.api.annotation.Plugin;
23+
import io.cdap.cdap.api.data.format.StructuredRecord;
24+
import io.cdap.cdap.api.data.schema.Schema;
25+
import io.cdap.cdap.api.plugin.PluginConfig;
26+
import io.cdap.cdap.api.spark.sql.DataFrames;
27+
import io.cdap.cdap.etl.api.PipelineConfigurer;
28+
import io.cdap.cdap.etl.api.StageConfigurer;
29+
import io.cdap.cdap.etl.api.batch.SparkCompute;
30+
import io.cdap.cdap.etl.api.batch.SparkExecutionPluginContext;
3131
import org.apache.spark.api.java.JavaRDD;
3232
import org.apache.spark.api.java.function.Function;
3333
import org.apache.spark.sql.Row;

0 commit comments

Comments
 (0)