scala-sparkML学习笔记:scala解析json文件

2019-09-06 09:25:54 浏览数 (1)

版权声明:本文为博主原创文章,遵循 CC 4.0 BY-SA 版权协议,转载请附上原文出处链接和本声明。

本文链接:https://blog.csdn.net/u014365862/article/details/100163695

在搭建一些工程项目时,往往需要有配置文件,而配置文件很多是基于json格式组织的。

这里可直接看代码:

代码语言:javascript复制
import scala.util.parsing.json.JSON._
import scala.io.Source


object XMLHelloWorld {
  def main(args: Array[String]): Unit = {

    def regJson(json:Option[Any]) = json match {
      case Some(map: Map[String, Any]) => map
      //      case None => "erro"
      //      case other => "Unknow data structure : "   other
      }

    // 解析List
    def regList(tmp:Option[Any]) = tmp match {
        case Some( list: List[String] ) => list
    }

    // 解析String
    def regString(tmp:Option[Any]) = tmp match {
        case Some( str: String ) => str
    }


    var tt =  Map.empty[String, Any] 

    val tree = parseFull(Source.fromFile("config_params.json").mkString)
    
    val first = regJson(tree)
    println(first.get("experiment_name"))
    println( regString( first.get("experiment_name") ) )

    val dev = first.get("model_monitor")
    println( dev )
    val sec = regJson(dev)
    println( sec.get("evaluator") )
    val evList = regList( sec.get("evaluator") )
    println( enList )


    //tt = tree match {
    //  case Some(map: Map[String, Any]) => map
    //}
    //println(tt.getClass.getSimpleName)
    //println(tt.get("experiment_name"))
    //println(tt.get("model_monitor"))


  }
}
 

/*
// config_params.json
{
    "experiment_name": "qdml_command_test",
    "experiment_id": "16",
    "problem_type": "classifier",
    "estimator": "xgradient_boosting",
    "fit_gird": false,
    "eval_metric": "auc",
    "model_monitor": {
        "evaluator": [
            "auc",
            "precision_score",
            "recall_score",
            "ks_value"
        ],
        "threshold": 0.5,
        "is_vaild": true
    }
}
*/

pom.xml

代码语言:javascript复制
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>sample</groupId>
  <artifactId>scala-module-dependency-sample</artifactId>
  <version>1.0-SNAPSHOT</version>
  <!-- Maven profiles allow you to support both Scala 2.10, 2.11 and Scala 2.12 with
    the right dependencies for modules specified for each version separately -->
  <profiles>
    <profile>
      <id>scala-2.12</id>
      <activation>
        <activeByDefault>true</activeByDefault>
      </activation>
      <properties>
        <scalaVersion>2.12.0</scalaVersion>
        <scalaBinaryVersion>2.12</scalaBinaryVersion>
      </properties>
      <dependencies>
          <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>${scalaVersion}</version>
          </dependency>
          <dependency>
            <groupId>org.scala-lang.modules</groupId>
            <artifactId>scala-xml_${scalaBinaryVersion}</artifactId>
            <version>1.0.6</version>
          </dependency>
          <dependency>
            <groupId>org.scala-lang.modules</groupId>
            <artifactId>scala-parser-combinators_${scalaBinaryVersion}</artifactId>
            <version>1.0.4</version>
          </dependency>
          <dependency>
            <groupId>org.scala-lang.modules</groupId>
            <artifactId>scala-swing_${scalaBinaryVersion}</artifactId>
            <version>2.0.0-M2</version>
          </dependency>
          <dependency>
            <groupId>net.minidev</groupId>
            <artifactId>json-smart</artifactId>
            <version>2.3</version>
          </dependency>
      </dependencies>
    </profile>
    <profile>
      <id>scala-2.11</id>
      <properties>
        <scalaVersion>2.11.8</scalaVersion>
        <scalaBinaryVersion>2.11</scalaBinaryVersion>
      </properties>
      <dependencies>
          <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>${scalaVersion}</version>
          </dependency>
          <dependency>
            <groupId>org.scala-lang.modules</groupId>
            <artifactId>scala-xml_${scalaBinaryVersion}</artifactId>
            <version>1.0.6</version>
          </dependency>
          <dependency>
            <groupId>org.scala-lang.modules</groupId>
            <artifactId>scala-parser-combinators_${scalaBinaryVersion}</artifactId>
            <version>1.0.4</version>
          </dependency>
          <dependency>
            <groupId>org.scala-lang.modules</groupId>
            <artifactId>scala-swing_${scalaBinaryVersion}</artifactId>
            <version>1.0.2</version>
          </dependency>
      </dependencies>
    </profile>
    <profile>
      <id>scala-2.10</id>
      <properties>
        <scalaVersion>2.10.6</scalaVersion>
        <scalaBinaryVersion>2.10</scalaBinaryVersion>
      </properties>
      <dependencies>
          <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>${scalaVersion}</version>
          </dependency>
          <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-swing</artifactId>
            <version>${scalaVersion}</version>
          </dependency>
      </dependencies>
    </profile>
  </profiles>
  <build>
    <sourceDirectory>src/main/scala</sourceDirectory>
    <plugins>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-compiler-plugin</artifactId>
        <configuration>
          <source>1.6</source>
          <target>1.6</target>
        </configuration>
      </plugin>
      <plugin>
        <groupId>net.alchim31.maven</groupId>
        <artifactId>scala-maven-plugin</artifactId>
        <version>3.2.1</version>
        <executions>
          <execution>
            <goals>
              <goal>compile</goal>
              <goal>testCompile</goal>
            </goals>
          </execution>
        </executions>
        <configuration>
          <args>
            <!-- work-around for https://issues.scala-lang.org/browse/SI-8358 -->
            <arg>-nobootcp</arg>
          </args>
        </configuration>
      </plugin>
 
      <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-shade-plugin</artifactId>
          <version>1.2.1</version>
          <executions>
              <execution>
                  <phase>package</phase>
                  <goals>
                          <goal>shade</goal>
                  </goals>
                      <configuration>
                          <transformers>
                              <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                  <mainClass>XMLHelloWorld</mainClass>
                              </transformer>
                          </transformers>
                      </configuration>
              </execution>
          </executions>
       </plugin>
 
 
    </plugins>
  </build>
</project>

运行:

mvn clean package

cd target

java -jar scala-module-dependency-sample-1.0-SNAPSHOT.jar

结果:

Some(qdml_command_test) Some(Map(evaluator -> List(auc, precision_score, recall_score, ks_value), threshold -> 0.5, is_vaild -> true)) Some(List(auc, precision_score, recall_score, ks_value))

git:https://github.com/MachineLP/Spark-/tree/master/scala-json

还可以参考:(1)https://www.cnblogs.com/hithink/p/9986227.html

0 人点赞