Source: eclairjs/sql/SparkSession.js

  1. /*                                                                         
  2. * Copyright 2016 IBM Corp.                                                 
  3. *                                                                          
  4. * Licensed under the Apache License, Version 2.0 (the "License");          
  5. * you may not use this file except in compliance with the License.         
  6. * You may obtain a copy of the License at                                  
  7. *                                                                          
  8. *      http://www.apache.org/licenses/LICENSE-2.0                          
  9. *                                                                          
  10. * Unless required by applicable law or agreed to in writing, software      
  11. * distributed under the License is distributed on an "AS IS" BASIS,        
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
  13. * See the License for the specific language governing permissions and      
  14. * limitations under the License.                                           
  15. */ 
  17. (function () {
  21.     
  22.     /**
  23.      * @classdesc
  24.      * The entry point to programming Spark with the Dataset and DataFrame API.
  25.      *
  26.      * In environments that this has been created upfront (e.g. REPL, notebooks), use the builder
  27.      * to get an existing session:
  28.      *
  29.      * @example 
  30.      *   SparkSession.builder().getOrCreate()
  31.      *  
  32.      *
  33.      * The builder can also be used to create a new session:
  34.      *
  35.      * @example 
  36.      *   SparkSession.builder()
  37.      *     .master("local")
  38.      *     .appName("Word Count")
  39.      *     .config("spark.some.config.option", "some-value").
  40.      *     .getOrCreate()
  41.      *  
  42.      * @class
  43.      * @memberof module:eclairjs/sql
  44.      */
  45.     
  46.       var SparkSession = Java.type('org.eclairjs.nashorn.wrap.sql.SparkSession');
  48.     
  49.     
  50.     /**
  51.      * The version of Spark on which this application is running.
  52.      *
  53.      * @since EclairJS 0.6 Spark  2.0.0
  54.      * @function
  55.      * @name module:eclairjs/sql.SparkSession#version
  56.      * @returns {string}
  57.      */
  60.     /**
  61.      * The underlying SparkContext.
  62.      *
  63.      * @since EclairJS 0.6 Spark  2.0.0
  64.      * @function
  65.      * @name module:eclairjs/sql.SparkSession#sparkContext
  66.      * @returns {module:eclairjs.SparkContext}
  67.      */
  69.     /**
  70.      * A collection of methods for registering user-defined functions (UDF).
  71.      * Note that the user-defined functions must be deterministic. Due to optimization,
  72.      * duplicate invocations may be eliminated or the function may even be invoked more times than
  73.      * it is present in the query.
  74.      *
  75.      *  
  76.      *
  77.      * @since EclairJS 0.6 Spark  2.0.0
  78.      * @function
  79.      * @name module:eclairjs/sql.SparkSession#udf
  80.      * @returns {module:eclairjs/sql.UDFRegistration}
  81.      */
  82. /*
  83.     SparkSession.prototype.udf = function() {
  84.        var javaObject =  this.getJavaObject().udf();
  85.        return Utils.javaToJs(javaObject);
  86.     };
  87.     
  88. */
  90.     /**
  91.      * :: Experimental ::
  92.      * Returns a {@link StreamingQueryManager} that allows managing all the
  93.      * [[StreamingQuery StreamingQueries]] active on `this`.
  94.      *
  95.      * @since EclairJS 0.6 Spark  2.0.0
  96.      * @function
  97.      * @name module:eclairjs/sql.SparkSession#streams
  98.      * @returns {module:eclairjs/sql/streaming.StreamingQueryManager}
  99.      */
  100. /*
  101.     SparkSession.prototype.streams = function() {
  102.        var javaObject =  this.getJavaObject().streams();
  103.        return Utils.javaToJs(javaObject);
  104.     };
  105. */
  107.     
  108.     /**
  109.      * Start a new session with isolated SQL configurations, temporary tables, registered
  110.      * functions are isolated, but sharing the underlying {@link SparkContext} and cached data.
  111.      *
  112.      * Note: Other than the {@link SparkContext}, all shared state is initialized lazily.
  113.      * This method will force the initialization of the shared state to ensure that parent
  114.      * and child sessions are set up with the same shared state. If the underlying catalog
  115.      * implementation is Hive, this will initialize the metastore, which may take some time.
  116.      *
  117.      * @since EclairJS 0.6 Spark  2.0.0
  118.      * @function
  119.      * @name module:eclairjs/sql.SparkSession#newSession
  120.      * @returns {module:eclairjs/sql.SparkSession}
  121.      */
  122. /*    SparkSession.prototype.newSession = function() {
  123.        var javaObject =  this.getJavaObject().newSession();
  124.        return Utils.javaToJs(javaObject);
  125.     };
  126.     */
  127.     
  128.     /**
  129.      * :: Experimental ::
  130.      * Creates a new {@link Dataset} of type T containing zero elements.
  131.      *
  132.      * @function
  133.      * @name module:eclairjs/sql.SparkSession#emptyDataset
  134.      * @returns {module:eclairjs/sql.Dataset}  2.0.0
  135.      */
  136. /*
  137.     SparkSession.prototype.emptyDataset = function() {
  138.        var javaObject =  this.getJavaObject().emptyDataset();
  139.            return Utils.javaToJs(javaObject);
  141.     };
  142. */
  145.     /**
  146.      * Creates a {@link Dataset} from {@link RDD} of Rows using the schema
  147.      * @function
  148.      * @name module:eclairjs/sql.SparkSession#createDataFrame
  149.      * @param {module:eclairjs.RDD<module:eclairjs/sql.Row> | module:eclairjs/sql.Row[]} rowRDD_or_values A RDD of [Rows]{@link Row} or array of arrays that contain values of valid {@link DataTypes}
  150.      * @param {module:eclairjs/sql/types.StructType} schema -
  151.      * @returns {module:eclairjs/sql.DataFrame}
  152.      * @example
  153.      * var df = sqlSession.createDataFrame([[1,1], [1,2], [2,1], [2,1], [2,3], [3,2], [3,3]], schema);
  154.      *
  155.      */
  157.     /**
  158.     * Creates a {@link Dataset} from RDD of JSON
  159.     * @function
  160.     * @name module:eclairjs/sql.SparkSession#createDataFrameFromJson
  161.     * @param {{module:eclairjs.RDD<object>}    RDD of JSON
  162.     * @param {object} schema - object with keys corresponding to JSON field names (or getter functions), and values indicating Datatype
  163.     * @returns {module:eclairjs/sql.Dataset}
  164.     * @example
  165.     * var df = sqlSession.createDataFrameFromJson([{id:1,"name":"jim"},{id:2,"name":"tom"}], {"id":"Integer","name","String"});
  166.     *
  167.     */
  169.     /**
  170.      * Convert a [[BaseRelation]] created for external data sources into a {@link DataFrame}.
  171.      *
  172.      * @since EclairJS 0.6 Spark  2.0.0
  173.      * @function
  174.      * @name module:eclairjs/sql.SparkSession#baseRelationToDataFrame
  175.      * @param {module:eclairjs/sql/sources.BaseRelation} baseRelation
  176.      * @returns {DataFrame} 
  177.      */
  178. /*    SparkSession.prototype.baseRelationToDataFrame = function(baseRelation) {
  179.        var baseRelation_uw = Utils.unwrapObject(baseRelation);
  180.        var javaObject =  this.getJavaObject().baseRelationToDataFrame(baseRelation_uw);
  181.        return Utils.javaToJs(javaObject);
  182.     };*/
  183.     
  184. //
  185. //    /**
  186. //     * :: Experimental ::
  187. //     * Creates a {@link Dataset} from a local Seq of data of a given type. This method requires an
  188. //     * encoder (to convert a JVM object of type `T` to and from the internal Spark SQL representation)
  189. //     * that is generally created automatically through implicits from a `SparkSession`, or can be
  190. //     * created explicitly by calling static methods on {@link Encoders}.
  191. //     *
  192. //     * == Example ==
  193. //     *
  194. //     * @example
  195. //     *
  196. //     *   import spark.implicits._
  197. //     *   case class Person(name: String, age: Long)
  198. //     *   val data = Seq(Person("Michael", 29), Person("Andy", 30), Person("Justin", 19))
  199. //     *   val ds = spark.createDataset(data)
  200. //     *
  201. //     *   ds.show()
  202. //     *   // +-------+---+
  203. //     *   // |   name|age|
  204. //     *   // +-------+---+
  205. //     *   // |Michael| 29|
  206. //     *   // |   Andy| 30|
  207. //     *   // | Justin| 19|
  208. //     *   // +-------+---+
  209. //     *
  210. //     *
  211. //     * @since EclairJS 0.6 Spark  2.0.0
  212. //     * @param {object[]} data
  213. //     * @returns {module:eclairjs/sql.Dataset}
  214. //     */
  215. //    SparkSession.prototype.createDataset0 = function(data) {
  216. //    throw "not implemented by ElairJS";
  217. //    //   var data_uw = Utils.unwrapObject(data);
  218. //    //   var javaObject =  this.getJavaObject().createDataset(data_uw);
  219. //    //   return new Dataset(javaObject);
  220. //    };
  221. //
  222. //
  223. //    /**
  224. //     * :: Experimental ::
  225. //     * Creates a {@link Dataset} from an RDD of a given type. This method requires an
  226. //     * encoder (to convert a JVM object of type `T` to and from the internal Spark SQL representation)
  227. //     * that is generally created automatically through implicits from a `SparkSession`, or can be
  228. //     * created explicitly by calling static methods on {@link Encoders}.
  229. //     *
  230. //     * @since EclairJS 0.6 Spark  2.0.0
  231. //     * @param {module:eclairjs/rdd.RDD} data
  232. //     * @returns {module:eclairjs/sql.Dataset}
  233. //     */
  234. //    SparkSession.prototype.createDataset1 = function(data) {
  235. //    throw "not implemented by ElairJS";
  236. //    //   var data_uw = Utils.unwrapObject(data);
  237. //    //   var javaObject =  this.getJavaObject().createDataset(data_uw);
  238. //    //   return new Dataset(javaObject);
  239. //    };
  240. //
  241. //
  242. //    /**
  243. //     * :: Experimental ::
  244. //     * Creates a [[Dataset]] from a {@link List} of a given type. This method requires an
  245. //     * encoder (to convert a JVM object of type `T` to and from the internal Spark SQL representation)
  246. //     * that is generally created automatically through implicits from a `SparkSession`, or can be
  247. //     * created explicitly by calling static methods on {@link Encoders}.
  248. //     *
  249. //     * == Java Example ==
  250. //     *
  251. //     * @example
  252. //     *     List<String> data = Arrays.asList("hello", "world");
  253. //     *     Dataset<String> ds = spark.createDataset(data, Encoders.STRING());
  254. //     *
  255. //     *
  256. //     * @since EclairJS 0.6 Spark  2.0.0
  257. //     * @param {[]} data
  258. //     * @returns {module:eclairjs/sql.Dataset}
  259. //     */
  260. //    SparkSession.prototype.createDataset2 = function(data) {
  261. //    throw "not implemented by ElairJS";
  262. //    //   var javaObject =  this.getJavaObject().createDataset(data);
  263. //    //   return new Dataset(javaObject);
  264. //    };
  265.     
  267.     
  268.     /**
  269.      * :: Experimental ::
  270.      * Creates a [[Dataset]] with a single {@link LongType} column named `id`, containing elements
  271.      * in a range from `start` to `end` (exclusive) with a step value, with partition number
  272.      * specified.
  273.      *
  274.      * @since EclairJS 0.6 Spark  2.0.0
  275.      * @function
  276.      * @name module:eclairjs/sql.SparkSession#range
  277.      * @param {string} tableName
  278.      * @param {number} start
  279.      * @param {number} end
  280.      * @param {number} [step]
  281.      * @param {number} [numPartitions]
  282.      * @returns {module:eclairjs/sql.Dataset} 
  283.      */
  284. /*
  285.     SparkSession.prototype.range3 = function(start,end,step,numPartitions) {
  286.     throw "not implemented by ElairJS";
  287.     //   var javaObject =  this.getJavaObject().range(start,end,step,numPartitions);
  288. //       return Utils.javaToJs(javaObject);
  289.     };
  290. */
  292.     
  293.     /**
  294.      * Returns the specified table as a {@link module:eclairjs/sql.Dataset}.
  295.      *
  296.      * @since EclairJS 0.6 Spark  2.0.0
  297.      * @function
  298.      * @name module:eclairjs/sql.SparkSession#table
  299.      * @param {string} tableName
  300.      * @returns {module:eclairjs/sql.Dataset}
  301.      */
  302. /*
  303.     SparkSession.prototype.table = function(tableName) {
  304.        var javaObject =  this.getJavaObject().table(tableName);
  305.        return Utils.javaToJs(javaObject);
  306.     };
  307. */
  309.     
  310.     /**
  311.      * Executes a SQL query using Spark, returning the result as a {@link module:eclairjs/sql.Dataset}.
  312.      * The dialect that is used for SQL parsing can be configured with 'spark.sql.dialect'.
  313.      *
  314.      * @since EclairJS 0.6 Spark  2.0.0
  315.      * @function
  316.      * @name module:eclairjs/sql.SparkSession#sql
  317.      * @param {string} sqlText
  318.      * @returns {module:eclairjs/sql.Dataset}
  319.      */
  320. /*
  321.     SparkSession.prototype.sql = function(sqlText) {
  322.        var javaObject =  this.getJavaObject().sql(sqlText);
  323.        return Utils.javaToJs(javaObject);
  325.     };
  326. */
  328.     
  329.     /**
  330.      * Returns a {@link DataFrameReader} that can be used to read non-streaming data in as a
  331.      * {@link DataFrame}.
  332.      * @example 
  333.      *   sparkSession.read.parquet("/path/to/file.parquet")
  334.      *   sparkSession.read.schema(schema).json("/path/to/file.json")
  335.      *  
  336.      *
  337.      * @since EclairJS 0.6 Spark  2.0.0
  338.      * @function
  339.      * @name module:eclairjs/sql.SparkSession#read
  340.      * @returns {module:eclairjs/sql.DataFrameReader}
  341.      */
  342. /*
  343.     SparkSession.prototype.read = function() {
  344.        var javaObject =  this.getJavaObject().read();
  345.        return new DataFrameReader(javaObject);
  346.     };
  347. */
  349.     
  350.     /**
  351.      * :: Experimental ::
  352.      * Returns a [[DataStreamReader]] that can be used to read streaming data in as a {@link DataFrame}.
  353.      * @example 
  354.      *   sparkSession.readStream.parquet("/path/to/directory/of/parquet/files")
  355.      *   sparkSession.readStream.schema(schema).json("/path/to/directory/of/json/files")
  356.      *  
  357.      *
  358.      * @since EclairJS 0.6 Spark  2.0.0
  359.      * @function
  360.      * @name module:eclairjs/sql.SparkSession#readStream
  361.      * @returns {module:eclairjs/sql/streaming.DataStreamReader}
  362.      */
  363. /*    SparkSession.prototype.readStream = function() {
  364.        var javaObject =  this.getJavaObject().readStream();
  365.        return Utils.javaToJs(javaObject);
  366.     };
  367.     */
  368.     
  369.     /**
  370.      * Stop the underlying {@link module:eclairjs.SparkContext}.
  371.      *
  372.      * @function
  373.      * @name module:eclairjs/sql.SparkSession#stop
  374.      * @since EclairJS 0.6 Spark  2.0.0
  375.      */
  377.     //
  378.     // static methods
  379.     //
  380.     
  381.     
  382.     /**
  383.      * Creates a [[module:eclairjs/sql.SparkSessionBuilder]] for constructing a {@link module:eclairjs/sql.SparkSession}.
  384.      *
  385.      * @since EclairJS 0.6 Spark  2.0.0
  386.      * @function
  387.      * @static
  388.      * @name module:eclairjs/sql.SparkSession#builder
  389.      * @returns {module:eclairjs/sql.SparkSessionBuilder}
  390.      */
  392.     
  393.     /**
  394.      * Changes the SparkSession that will be returned in this thread and its children when
  395.      * SparkSession.getOrCreate() is called. This can be used to ensure that a given thread receives
  396.      * a SparkSession with an isolated session, instead of the global (first created) context.
  397.      *
  398.      * @since EclairJS 0.6 Spark  2.0.0
  399.      * @function
  400.      * @static
  401.      * @name module:eclairjs/sql.SparkSession#setActiveSession
  402.      * @param {module:eclairjs/sql.SparkSession} session
  403.      */
  405.     
  406.     /**
  407.      * Clears the active SparkSession for current thread. Subsequent calls to getOrCreate will
  408.      * return the first created context instead of a thread-local override.
  409.      *
  410.      * @since EclairJS 0.6 Spark  2.0.0
  411.      * @function
  412.      * @static
  413.      * @name module:eclairjs/sql.SparkSession#clearActiveSession
  414.      */
  416.     
  417.     /**
  418.      * Sets the default SparkSession that is returned by the builder.
  419.      *
  420.      * @since EclairJS 0.6 Spark  2.0.0
  421.      * @function
  422.      * @static
  423.      * @name module:eclairjs/sql.SparkSession#setDefaultSession
  424.      * @param {module:eclairjs/sql.SparkSession} session
  425.      */
  427.     
  428.     /**
  429.      * Clears the default SparkSession that is returned by the builder.
  430.      *
  431.      * @function
  432.      * @static
  433.      * @name module:eclairjs/sql.SparkSession#clearDefaultSession
  434.      * @since EclairJS 0.6 Spark  2.0.0
  435.      */
  437.     module.exports = SparkSession;
  438. })();