1、下载
http://spark.apache.org/downloads.html
https://www.apache.org/dyn/closer.lua/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz
2、解压配置
（1）解压
代码语言：javascript复制[root@node2 app]# ls
nacos spark-3.1.2-bin-hadoop3.2.tgz
[root@node2 app]# tar -zxvf spark-3.1.2-bin-hadoop3.2.tgz（2）重命名
代码语言：javascript复制[root@node2 app]# mv spark-3.1.2-bin-hadoop3.2 spark-3.1.2
[root@node2 app]# cd spark-3.1.2/
[root@node2 spark-3.1.2]# ls
bin conf data examples jars kubernetes LICENSE licenses NOTICE python R README.md RELEASE sbin yarn
[root@node2 spark-3.1.2]# ll（3）配置workers（旧版本是slaves）
代码语言：javascript复制[root@node2 spark-3.1.2]# cd conf/
[root@node2 conf]# ls
fairscheduler.xml.template log4j.properties.template metrics.properties.template spark-defaults.conf.template spark-env.sh.template workers.template
[root@node2 conf]# vi workers.template
[root@node2 conf]# cp workers.template workers
[root@node2 conf]# vi workers
[root@node2 conf]# cat workers
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# A Spark Worker will be started on each of the machines listed below.
node1
node2
node3
[root@node2 conf]# （4）配置spark-env.sh
代码语言：javascript复制[root@node2 conf]# cp spark-env.sh.template spark-env.sh
[root@node2 conf]# vi spark-env.sh添加一行
代码语言：javascript复制export JAVA_HOME=/opt/jdk1.8.0_161（5）分发
代码语言：javascript复制[root@node2 app]# scp -r spark-3.1.2 node1:/app
[root@node2 app]# scp -r spark-3.1.2 node3:/app3、启动集群
代码语言：javascript复制[root@node1 app]# cd spark-3.1.2
[root@node1 spark-3.1.2]# sbin/start-master.sh
starting org.apache.spark.deploy.master.Master, logging to /app/spark-3.1.2/logs/spark-root-org.apache.spark.deploy.master.Master-1-node1.out
[root@node1 spark-3.1.2]# jps
6148 PaloFe
11689 Jps
11325 Master
[root@node1 spark-3.1.2]# sbin/start-slaves.sh
This script is deprecated, use start-workers.sh
node1: starting org.apache.spark.deploy.worker.Worker, logging to /app/spark-3.1.2/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-node1.out
node3: starting org.apache.spark.deploy.worker.Worker, logging to /app/spark-3.1.2/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-node3.out
node2: starting org.apache.spark.deploy.worker.Worker, logging to /app/spark-3.1.2/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-node2.out
[root@node1 spark-3.1.2]# jps
6148 PaloFe
14983 Jps
14204 Worker
11325 Master
[root@node1 spark-3.1.2]# [root@node2 app]# curl node1:8080
<!DOCTYPE html><html>
<head>
<meta http-equiv="Content-type" content="text/html; charset=utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/static/bootstrap.min.css" type="text/css"/><link rel="stylesheet" href="/static/vis-timeline-graph2d.min.css" type="text/css"/><link rel="stylesheet" href="/static/webui.css" type="text/css"/><link rel="stylesheet" href="/static/timeline-view.css" type="text/css"/><script src="/static/sorttable.js"></script><script src="/static/jquery-3.5.1.min.js"></script><script src="/static/vis-timeline-graph2d.min.js"></script><script src="/static/bootstrap.bundle.min.js"></script><script src="/static/initialize-tooltips.js"></script><script src="/static/table.js"></script><script src="/static/timeline-view.js"></script><script src="/static/log-view.js"></script><script src="/static/webui.js"></script><script>setUIRoot('')</script>
<link rel="shortcut icon" href="/static/spark-logo-77x50px-hd.png"></link>
<title>Spark Master at spark://node1:7077</title>
</head>
<body>
<div class="container-fluid">
<div class="row">
<div class="col-12">
<h3 style="vertical-align: middle; display: inline-block;">
<a style="text-decoration: none" href="/">
<img src="/static/spark-logo-77x50px-hd.png"/>
<span class="version" style="margin-right: 15px;">3.1.2</span>
</a>
Spark Master at spark://node1:7077
</h3>
</div>
</div>
<div class="row">
<div class="col-12">
<div class="row">
<div class="col-12">
<ul class="list-unstyled">
<li><strong>URL:</strong> spark://node1:7077</li>
<li><strong>Alive Workers:</strong> 3</li>
<li><strong>Cores in use:</strong> 12 Total,
0 Used</li>
<li><strong>Memory in use:</strong>
19.9 GiB Total,
0.0 B Used</li>
<li><strong>Resources in use:</strong>
</li>
<li><strong>Applications:</strong>
0 <a href="#running-app">Running</a>,
0 <a href="#completed-app">Completed</a> </li>
<li><strong>Drivers:</strong>
0 Running,
0 Completed </li>
<li><strong>Status:</strong> ALIVE</li>
</ul>
</div>
</div><div class="row">
<div class="col-12">
<span class="collapse-aggregated-workers collapse-table" onClick="collapseTable('collapse-aggregated-workers','aggregated-workers')">
<h4>
<span class="collapse-table-arrow arrow-open"></span>
<a>Workers (3)</a>
</h4>
</span>
<div class="aggregated-workers collapsible-table">
<table class="table table-bordered table-sm table-striped sortable">
<thead><th width="" class="">Worker Id</th><th width="" class="">Address</th><th width="" class="">State</th><th width="" class="">Cores</th><th width="" class="">Memory</th><th width="" class="">Resources</th></thead>
<tbody>
<tr>
<td>
<a href="http://10.17.12.158:8081">
worker-20210811170616-10.17.12.158-36757
</a>
</td>
<td>10.17.12.158:36757</td>
<td>ALIVE</td>
<td>4 (0 Used)</td>
<td sorttable_customkey="6799.0">
6.6 GiB
(0.0 B Used)
</td>
<td></td>
</tr><tr>
<td>
<a href="http://10.17.12.159:8081">
worker-20210811170617-10.17.12.159-33329
</a>
</td>
<td>10.17.12.159:33329</td>
<td>ALIVE</td>
<td>4 (0 Used)</td>
<td sorttable_customkey="6799.0">
6.6 GiB
(0.0 B Used)
</td>
<td></td>
</tr><tr>
<td>
<a href="http://10.17.12.160:8081">
worker-20210811170617-10.17.12.160-40490
</a>
</td>
<td>10.17.12.160:40490</td>
<td>ALIVE</td>
<td>4 (0 Used)</td>
<td sorttable_customkey="6799.0">
6.6 GiB
(0.0 B Used)
</td>
<td></td>
</tr>
</tbody>
</table>
</div>
</div>
</div><div class="row">
<div class="col-12">
<span id="running-app" class="collapse-aggregated-activeApps collapse-table" onClick="collapseTable('collapse-aggregated-activeApps','aggregated-activeApps')">
<h4>
<span class="collapse-table-arrow arrow-open"></span>
<a>Running Applications (0)</a>
</h4>
</span>
<div class="aggregated-activeApps collapsible-table">
<table class="table table-bordered table-sm table-striped sortable">
<thead><th width="" class="">Application ID</th><th width="" class="">Name</th><th width="" class="">Cores</th><th width="" class="">Memory per Executor</th><th width="" class="">Resources Per Executor</th><th width="" class="">Submitted Time</th><th width="" class="">User</th><th width="" class="">State</th><th width="" class="">Duration</th></thead>
<tbody>
</tbody>
</table>
</div>
</div>
</div><div>
</div><div class="row">
<div class="col-12">
<span id="completed-app" class="collapse-aggregated-completedApps collapse-table" onClick="collapseTable('collapse-aggregated-completedApps',
'aggregated-completedApps')">
<h4>
<span class="collapse-table-arrow arrow-open"></span>
<a>Completed Applications (0)</a>
</h4>
</span>
<div class="aggregated-completedApps collapsible-table">
<table class="table table-bordered table-sm table-striped sortable">
<thead><th width="" class="">Application ID</th><th width="" class="">Name</th><th width="" class="">Cores</th><th width="" class="">Memory per Executor</th><th width="" class="">Resources Per Executor</th><th width="" class="">Submitted Time</th><th width="" class="">User</th><th width="" class="">State</th><th width="" class="">Duration</th></thead>
<tbody>
</tbody>
</table>
</div>
</div>
</div><div>
</div>
</div>
</div>
</div>
</body>
</html>[root@node2 app]# 
