一、 REST API
1. 相关参考
YARN REST API 总结-阿里云开发者社区
Apache Hadoop 3.3.6 – Hadoop YARN - Introduction to the web services REST APIs
2. python获取应用信息
import requests
from functools import reduce
import json
import os
# 激活状态的app信息
def get_apps(yarn_url):
# 访问URL
url = "http://%s:8088/ws/v1/cluster/apps"%(yarn_url)
print('get_apps访问url:' + url)
# 参数
params = {
#"minEndDate": min_end_date,
#"maxEndDate": max_end_date,
"state": "running",
#"limit": "3",
}
return requests.get(url, params = params, timeout = 30).json()['apps']['app']
# 格式化并过滤app
def format_filter_apps(apps, timeout):
format_apps = []
for app in apps:
if app['state'] == 'RUNNING' and app['elapsedTime'] > timeout and app['applicationType'].lower() == 'spark' :
format_apps.append({
'id' : app['id'],
'user' : app['user'],
'name' : app['name'],
'user' : app['user'],
'type' : app['applicationType'],
'start_time' : app['startedTime'],
'state' : app['state'],
'queue' : app['queue'],
'elapsed_time' : app['elapsedTime'],
'allocated_mb' : app['allocatedMB'],
'reserved_mb' : app['reservedMB'],
'running_containers' : app['runningContainers'],
})
return format_apps
# 设定参数
yarn_url='aa.bb.cc.dd'
timeout=30*60*1000
# 获取结果
format_apps=format_filter_apps( get_apps(yarn_url),timeout)
for app in format_apps:
print(app)