常见操作
保存容器
docker export containID > filename
保存、加载镜像
docker save imageID > filename
docker load < filename
$ docker pull centos:6 # 下载镜像
6: Pulling from library/centos
1c8f9aa56c90: Pull complete
Digest: sha256:b358c4a16ef77db3a07eaaaf62c707f51aa15bca820489392cc9d97046bc483a
Status: Downloaded newer image for centos:6
$ docker image ls # 列出本地镜像
REPOSITORY TAG IMAGE ID CREATED SIZE
centos 6 b5e5ffb5cdea 4 weeks ago 194MB
elasticsearch 2 d859b9416a9f 5 months ago 563MB
vuls/goval-dictionary latest e4ff94bb7ae7 6 months ago 971MB
vuls/vuls latest 18dfbb718f5c 6 months ago 1.46GB
vuls/go-cve-dictionary latest bd500592c758 6 months ago 1.03GB
graylog2/server latest 5c2e160f7976 7 months ago 597MB
mongo 2 1999482cb0a5 22 months ago 391MB
$ docker image rm e4ff94bb7ae7 18dfbb718f5c bd500592c758 # 删除本地镜像
$ docker run -dit centos:6 # 运行centos容器，并保持运行状态
d29c70dc73fa7cff2f6746b66fbd1b9e405427fee46a0a4ad7b920da9654d1c1
$ docker container ls # 列出运行中的容器
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
d29c70dc73fa centos:6 "/bin/bash" 22 seconds ago Up 11 seconds relaxed_booth
$ docker exec -i d29c70dc73fa /bin/echo "xxxx" # 执行命令
xxxx
$ docker exec -t -i d29c70dc73fa /bin/bash # 分配伪终端
[root@d29c70dc73fa /]# ls -la
$ docker container stop d29c70dc73fa # 终止容器
安装apache hadoop
https://hadoop.apache.org/docs/r3.0.3/hadoop-project-dist/hadoop-common/SingleCluster.html
[root@d29c70dc73fa bin]# wget http://www-us.apache.org/dist/hadoop/common/stable/hadoop-2.9.1.tar.gz
[root@d29c70dc73fa bin]# tar zxvf hadoop-2.9.1.tar.gz
export HADOOP_HOME='/usr/local/bin/hadoop'
export PATH=$HADOOP_HOME/bin:$PATH
# 安装java
$ yum install java
export JAVA_HOME='/usr'
安装Hive
https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Cli#LanguageManualCli-HiveResources
[root@d29c70dc73fa bin]# wget http://www-eu.apache.org/dist/hive/stable-2/apache-hive-2.3.3-bin.tar.gz
[root@d29c70dc73fa apache-hive-2.3.3-bin]# tar -xzvf apache-hive-2.3.3-bin.tar.gz
[root@d29c70dc73fa apache-hive-2.3.3-bin]# cd apache-hive-2.3.3-bin
[root@d29c70dc73fa apache-hive-2.3.3-bin]# export HIVE_HOME=`pwd`
[root@d29c70dc73fa apache-hive-2.3.3-bin]# export PATH=$HIVE_HOME/bin:$PATH
[root@d29c70dc73fa apache-hive-2.3.3-bin]# vi /etc/profile
[root@d29c70dc73fa apache-hive-2.3.3-bin]# $HIVE_HOME/bin/hive
Cannot find hadoop installation: $HADOOP_HOME or $HADOOP_PREFIX must be set or hadoop must be in the path
# 检查hive的配置文件
加载elasticsearch-hadoop包
https://www.elastic.co/downloads/hadoop https://www.elastic.co/guide/en/elasticsearch/hadoop/current/hive.html#hive
$ wget https://artifacts.elastic.co/downloads/elasticsearch-hadoop/elasticsearch-hadoop-6.4.0.zip
$ hive
hive> ADD JAR /usr/local/bin/elasticsearch-hadoop-6.4.0/dist/elasticsearch-hadoop-6.4.0.jar;
hive> ADD JAR /usr/local/bin/elasticsearch-hadoop-6.4.0/dist/elasticsearch-hadoop-hive-6.4.0.jar;
# 测试数据
CREATE EXTERNAL TABLE es_alexa_top_1 (`rank_id` int, `domain` string)
STORED BY 'org.elasticsearch.hadoop.hive.EsStorageHandler'
TBLPROPERTIES('es.index.auto.create' = 'true','es.resource' = 'alexa_top/list', 'es.nodes'='172.17.0.3','es.port'='9200');
SET hive.mapred.reduce.tasks.speculative.execution = false;
SET mapreduce.map.speculative = false;
SET mapreduce.reduce.speculative = false;
INSERT overwrite TABLE es_alexa_top_1
SELECT
`rank_id`,
`domain`
FROM alexa_top;
- Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient
- 删除metastore_db后，重建
- find / -name metastore_db
- 在hive里执行重建
[root@d29c70dc73fa apache-hive-2.3.3-bin]# ./bin/schematool -initSchema -dbType derby
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/usr/bin/apache-hive-2.3.3-bin/lib/log4j-slf4j-impl-2.6.2.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/usr/bin/hadoop-2.9.1/share/hadoop/common/lib/slf4j-log4j12-1.7.25.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
Metastore connection URL: jdbc:derby:;databaseName=metastore_db;create=true
Metastore Connection Driver : org.apache.derby.jdbc.EmbeddedDriver
Metastore connection User: APP
Starting metastore schema initialization to 2.3.0
Initialization script hive-schema-2.3.0.derby.sql
Initialization script completed
schemaTool completed
启动hive
# 启动hive的时候，最好在同一个目录下，否则会出现“Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient”报错
安装elasticsearch
FROM elasticsearch:2
RUN /usr/share/elasticsearch/bin/plugin install mobz/elasticsearch-head
EXPOSE 9200
$ # docker build -t happytree/es .
$ docker run -d -p 9200:9200 --name="es4" happytree/es
访问 http://127.0.0.1:9200/_plugin/head/
安装HUE
$ docker pull gethue/hue
$ docker run -it -p 8888:8888 gethue/hue:latest bash
$ ./build/env/bin/hue runserver_plus 0.0.0.0:8888
创建账号admin/admin
开启Hive Remote Server
hive --service hiveserver2 &
hive --service metastore &
beeline连接
bin/beeline
beeline> !connect jdbc:hive2://localhost:10000 admin admin
每次保存、运行容器
docker export containID > filename # 保存容器到本地
docker import filename [newname] # 加载本地容器，为image
# 加载、启动各个image
# hive
docker import --change 'CMD ["/bin/bash"]' myhive myhive
docker run -dit --name "container_hive" myhive
docker exec -t -i container_hive /bin/bash
# es
docker container start es4
docker exec -t -i es4 bash
# hue
# docker import --change 'CMD ["bash"]' myhue myhue
# docker run -it -p 8888:8888 --name "container_hue" myhue
docker container start container_hue
docker exec -t -i container_hue bash
/hue/build/env/bin/hue runserver_plus 0.0.0.0:8888
