ok
Mini Shell
# -*- coding: utf-8 -*-
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2022 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
#
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
import io
import stat
import re
import pwd
import os
import sys
import grp
import subprocess
import platform
import datetime
from functools import lru_cache
from configparser import ConfigParser, Error
from typing import Dict, Any, AnyStr, Optional, Tuple, Union # NOQA
import psutil
from lxml import etree
import secureio
from clcommon.lib.cledition import is_ubuntu
# These commands used to be in this file, but is_ubuntu cannot be imported if they're here
# due to a circular import with clcommon.lib.__init__ -> clcommon.lib.mysql_governor_lib.
# To avoid that, we move them to a separate file and import into this namespace
# so that things like `from clcommon.utils import run_command` still work.
from clcommon.utils_cmd import ( # NOQA
check_command,
run_command,
exec_utility,
ExternalProgramFailed,
)
# Not all imports are used in this file - they are made so that other
# modules can import them from utils, not utils_cmd. Therefore, NOQA.
RHN_SYSTEMID_FILE = "/etc/sysconfig/rhn/systemid"
WEEK_DAYS = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun")
LITESPEED_STATUS_FOR_USERS = "/opt/cloudlinux/litespeed_status"
def create_symlink(link_value, link_path):
"""
Create symlink link_path -> link_value if it does not exist or
points to different location
:param link_value: path that symlink should point to (symlink value)
:type link_value: str
:param link_path: path where to create symlink
:type link_path: str
"""
link_to = None
if os.path.islink(link_path):
try:
link_to = os.readlink(link_path)
except OSError:
pass
if link_value != link_to:
try:
os.unlink(link_path)
except OSError:
pass
os.symlink(link_value, link_path)
def get_file_lines(path, unicode_errors_handle=None):
"""
Read file and return file's lines
errors param may be passed to define how handle
unicode errors, errors=None is default value of open()
:param path: path to file
:param unicode_errors_handle: how to handle unicode errors
:return: list of file's lines
"""
content = []
if os.path.isfile(path):
with open(path, "r", errors=unicode_errors_handle) as f:
content = f.readlines()
return content
def write_file_lines(path, content, mode):
"""
Write lines to file
:param content: list of lines for writing to file
:param path: path to file
:param mode: open mode
:return: None
"""
with open(path, mode) as f:
f.writelines(content)
def delete_line_from_file(path, line):
"""
Delete line from file. Return True when line(s) have been deleted, False otherwise (specified line is not found)
:param path: path to file
:type path: string
:param line: line to delete without EOL ('\n')
:type line: string
:rtype bool
"""
file_lines = get_file_lines(path)
out_file_lines = [item for item in file_lines if line != item.rstrip("\n")]
found = len(file_lines) != len(out_file_lines)
write_file_lines(path, out_file_lines, "w+")
return found
def is_root_or_exit():
"""
Check whether current user is effectively root and exit if not
"""
euid = os.geteuid()
if euid != 0:
try:
# Additional admins placed in this special group
# by lvemanager hooks to add root-like privileges to them
gr_gid = grp.getgrnam("clsupergid").gr_gid
if gr_gid in os.getgroups() or os.getegid() == gr_gid:
return
except KeyError:
pass # No group - no privileges
print("Error: root privileges required. Abort.", file=sys.stderr)
sys.exit(-1)
def is_ea4():
"""
Detects is EA4 installed
:return: True - EA4 present; False - EA4 absent
"""
return os.path.isfile("/etc/cpanel/ea4/is_ea4")
def grep(
pattern,
path=None,
fixed_string=False,
match_any_position=True,
multiple_search=False,
data_from_file=None,
):
"""
Grep pattern in file
:param multiple_search: if True - search all match,
False - search first match
:param pattern: pattern for search
:param path: path to file
:param data_from_file: read data from file for parsing
:param fixed_string: if True - search only fixed string,
False - search by regexp
:param match_any_position: if True - search any match position,
False - search only from string begin
:return: Generator with matched strings
"""
if data_from_file is None:
data_from_file = get_file_lines(path)
result = None
if not fixed_string:
# It's append the symbol ^ to the regexp
# if we are searching from the begin of a string and by the regexp
if not pattern.startswith("^") and not match_any_position:
pattern = "^{}".format(pattern)
pattern_comp = re.compile(pattern)
else:
pattern_comp = None
for line in data_from_file:
if fixed_string:
if match_any_position and line.find(pattern) != -1:
result = line
elif line.startswith(pattern):
result = line
else:
if pattern_comp.search(line):
result = line
if multiple_search and result is not None:
yield result
elif result is not None:
break
result = None
if result is not None:
yield result
def _parse_systemid_file():
"""
:rtype: lxml.etree._ElementTree obj
"""
return etree.parse(RHN_SYSTEMID_FILE)
def get_rhn_systemid_value(name) -> Optional[str]:
"""
find a member in xml by name and return value
:type name: str
:rtype: str|None
"""
try:
rhn_systemid_xml = _parse_systemid_file()
for member in rhn_systemid_xml.iter("member"):
if member.find("name").text == name:
return member.find("value")[0].text
except (IOError, IndexError, KeyError, etree.ParseError):
return None
return None
def get_file_system_in_which_file_is_stored_on(file_path):
# type: (str) -> Dict[str, Any]
"""
This function is written for detect file system in which file is stored on.
E.g., the file can be stored in NFS and this can affect the normal operation of the file.
We want to receive information about FS in emergency situations during reading or writing
:param file_path: path to file, for which we want to detect file system
:return: dict, which contains two keys:
key 'success' can be equals to False if we got error or True if we got normal result
key 'details' can contais error string if key 'success' is False or result if key 'success' is True
"""
result = {
"success": False,
"details": 'File "%s" isn\'t exists' % file_path,
}
if not os.path.exists(file_path):
return result
# Command: mount | grep "on $(df <file_name> | tail -n 1 | awk '{print $NF}') type"
# Result: /usr/tmpDSK on /var/tmp type ext3 (rw,nosuid,noexec,relatime,data=ordered)
try:
mount_point = subprocess.check_output(
["df %s | tail -n 1 | awk '{print $NF}'" % file_path],
shell=True,
executable="/bin/bash",
text=True,
).strip()
data = subprocess.check_output(
['mount | grep "on %s type"' % mount_point],
shell=True,
executable="/bin/bash",
text=True,
).strip()
result["success"] = True
result["details"] = data
except (subprocess.CalledProcessError, OSError) as err:
data = 'We can\'t get file system for file "%s". Exception "%s"' % (
file_path,
err,
)
result["details"] = data
return result
def is_testing_enabled_repo() -> bool:
"""
Checks if testing is enabled in /etc/yum.repos.d/cloudlinux.repo config
:return: bool value if testing enabled or not
"""
parser = ConfigParser(interpolation=None, strict=False)
try:
parser.read("/etc/yum.repos.d/cloudlinux.repo")
res = parser.getboolean("cloudlinux-updates-testing", "enabled")
# use base exception for config parser class
except Error:
res = False
return res
def get_cl_version() -> Optional[str]:
"""
Returns cl version taking into account release version
E.g: release =
2.6.32-896.16.1.lve1.4.54.el6.x86_64
5.14.0-162.6.1.el9_1.x86_64
el6 = cl6
el8 = cl8
........
ubuntu+cl_extesions = cl8
:return appropriate version string
"""
if is_ubuntu():
return "cl8"
check_vals_decoder = {
"el6.": "cl6",
"el6h.": "cl6h",
"el7.": "cl7",
"el7h.": "cl7h",
"el8.": "cl8",
".el9": "cl9", # note the dot at the beginning because of possible `el9_1` and so on
}
release: Optional[str] = platform.release()
for check_val in check_vals_decoder:
if release and check_val in release:
return check_vals_decoder[check_val]
release = get_rhn_systemid_value("os_release")
ret_val = None
if release is None:
ret_val = None
elif "6" in release:
ret_val = "cl6"
elif "7" in release:
ret_val = "cl7"
elif "8" in release:
ret_val = "cl8"
elif "9" in release:
ret_val = "cl9"
return ret_val
@lru_cache(maxsize=None)
def get_virt_type() -> Optional[str]:
"""
Returns virtualization type on current system.
It is reachable via virt-what utility.
E.g.: 'kvm', 'bhyve', 'openvz', 'qemu'
All acceptable outputs are listed here:
https://people.redhat.com/~rjones/virt-what/virt-what.txt
Output will be returned with at least two rows
Sample:
> kvm
>
Furthermore, there is a possibility for multiple text rows
Sample:
> xen
> xen-domU
That's why, the result will be taken from a first row.
If the output is empty, and there were no errors, the machine
is either non-virtual, or virt-what tool isn't familiar with it's
hypervisor. But the list of supported hypervisors and containers
covers all popular VMs types.
:return: virt_type - Optional[AnyStr]
- appropriate virtualization type string,
- 'physical' if there is no virtualization,
- None if there was an error
"""
try:
virt_what_output = run_command(["/usr/sbin/virt-what"]).strip()
except (subprocess.CalledProcessError, FileNotFoundError):
return None
# Check for the non-empty output - virtualization exists
if virt_what_output != "":
return virt_what_output.split("\n")[0]
else:
return "physical"
def check_pid(pid: int):
"""
Checks for a process existence by os.kill command
If os.kill will be used as os.kill(pid, 0), it will
just check for a presence of such PID
And if such pid can't be reached with kill method,
there will be raised OSError
"""
try:
os.kill(pid, 0)
except OSError:
return False
else:
return True
def is_process_running(pid_file_path: str) -> bool:
"""Check if process running using pid file
Arguments:
pid_file_path: path to the pid file of service
Returns:
bool: True or False
"""
if os.path.isfile(pid_file_path):
with open(pid_file_path) as f:
try:
return check_pid(int(f.read().strip()))
except ValueError:
pass
return False
def is_litespeed_running() -> bool:
"""
Detects that server works under Litespeed.
Note: be careful when modifying this method.
It is used in X-Ray, ask @dkavchuk or someone else from C-Projects team
for details.
return: True - LS working; False - LS not running (stopped or absent)
"""
if os.geteuid() == 0:
pid_file_path = "/tmp/lshttpd/lshttpd.pid"
return is_process_running(pid_file_path)
else:
if not os.path.exists(LITESPEED_STATUS_FOR_USERS):
return False
else:
with open(LITESPEED_STATUS_FOR_USERS) as f:
status = f.read().strip()
return status == "0"
def is_nginx_running() -> bool:
"""Check if nginx is running
Returns:
Bool: True or False
"""
pid_file_path = "/run/nginx.pid"
return is_process_running(pid_file_path)
def get_passenger_package_name():
"""
Return proper passenger package according to apache version
:rtype: str
"""
if is_ea4():
# Package that brings passenger on cPanel has different names on different systems:
# CL6 - ea-ruby24-mod_passenger
# CL7/8 - ea-ruby27-mod_passenger
# CL9 - ea-apache24-mod-passenger
# But they all provide "apache24-passenger" package
return "apache24-passenger"
return "alt-mod-passenger"
def is_package_installed(package_name):
"""
Checks that package installed on server
:param package_name: str
:rtype: bool
"""
try:
if is_ubuntu():
run_command(["dpkg", "-s", package_name])
else:
run_command(["rpm", "-q", package_name])
except ExternalProgramFailed:
return False
return True
def get_package_db_errors():
"""
Pick the package manager to check depending on the OS.
If we're not on Ubuntu, assume a CL variant with RPM.
:return: Error string, if any, None otherwise.
"""
if is_ubuntu():
return get_apt_db_errors()
else:
return get_rpm_db_errors()
def get_apt_db_errors():
"""
Check the dpkg DB as described in https://man7.org/linux/man-pages/man1/dpkg.1.html
See `--audit`.
:return: Error string, if any, None otherwise.
"""
try:
prc = subprocess.Popen(
["dpkg", "--audit"],
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
std_out, std_err = prc.communicate()
# Should return 1 if the audit fails.
if prc.returncode != 0:
# Check error
return f"dpkg audit error: {std_out}\n{std_err}."
except (OSError, IOError,) as err:
return str(err)
# There is no dpkg DB errors
return None
def get_rpm_db_errors():
"""
Check RPM DB consistency.
:return: None - No RPM DB errors
string_message - Error description
"""
doc_link = (
"https://cloudlinux.zendesk.com/hc/en-us/articles/"
"115004075294-Fix-rpmdb-Thread-died-in-Berkeley-DB-library"
)
try:
prc = subprocess.Popen(
["/bin/rpm", "--dbpath", "/var/lib/rpm", "--verifydb"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
std_out, std_err = prc.communicate()
if prc.returncode != 0:
# Check error
return f"RPM DB check error: {std_out}\n{std_err}.\nSee doc: {doc_link}"
except (OSError, IOError,) as err:
return str(err)
# There is no RPM DB errors
return None
def silence_stdout_until_process_exit():
"""
Upon process exit, Sentry sometimes prints:
Sentry is attempting to send 1 pending error messages
Waiting up to 10 seconds
Press Ctrl-C to quit
This causes broken JSON in output.
See also this issue: https://github.com/getsentry/raven-python/issues/904
"""
sys.stdout = io.StringIO()
sys.stderr = io.StringIO()
def mod_makedirs(path, mod):
"""
Create directories with desired permissions
Changed in version 3.7: The mode argument no longer affects
the file permission bits of newly-created intermediate-level directories.
Because it we use umask while creating dirs
:param mod: desired permissions
"""
inverted_mod = 0o777 - (mod & 0o777)
with secureio.set_umask(inverted_mod):
os.makedirs(path, mod)
def is_user_present(username: str) -> bool:
"""
Check user existence in the system
"""
try:
pwd.getpwnam(username)
except KeyError:
return False
return True
def is_uid_present(uid: int) -> bool:
"""
Check uid existence in the system
"""
try:
pwd.getpwuid(uid)
except KeyError:
return False
return True
def is_socket_file(path: AnyStr) -> Optional[bool]:
"""
Check that file by path is socket
"""
try:
mode = os.lstat(path).st_mode
except (FileNotFoundError, IOError, OSError):
return None
is_socket = stat.S_ISSOCK(mode)
return is_socket
def get_system_runlevel() -> int:
"""
Get number of system run level by command `runlevel`.
"""
output = subprocess.check_output(
"/sbin/runlevel",
shell=True,
executable="/bin/bash",
text=True,
)
# output: N 5
# there is N - previous value of runlevel, 5 is current runlevel
result = output.strip().split()[1]
# `S` means single-mode. Equals to level `1`
level = 1 if result == "S" else int(result)
return level
def _get_service_state_on_init_d_system(service_name: str) -> Tuple[bool, bool]:
"""
Returns state of a service (present and enabled) for init.d system.
Returns False, False if a service doesn't exist
Returns True, False if a service exists and it's not enabled
Returns True, True if a service exists and it's enabled
"""
runlevel = get_system_runlevel()
try:
# LANG=C parameter allows to use C programming language (en-US)
# locale instead of current, since there can be non-English
# results in the chkconfig output, while we search for the `on`
output = subprocess.check_output(
# the command return non-zero code if a service doesn't exist
f"LANG=C /sbin/chkconfig --list {service_name}",
shell=True,
executable="/bin/bash",
text=True,
)
except (subprocess.CalledProcessError, FileNotFoundError,):
return False, False
# split output:
# `cl_plus_sender 0:off 1:off 2:on 3:on 4:on 5:on 6:off`
output_list = output.strip().split()
for state_info in output_list[1:]:
state_list = state_info.strip().split(":")
is_active = state_list[1] == "on"
state_runlevel = int(state_list[0])
if runlevel == state_runlevel:
return True, is_active
return True, False
def _get_service_state_on_systemd_system(service_name: str) -> Tuple[bool, bool]:
"""
Returns state of service (present and enabled) for systemd system
Returns False, False if a service doesn't exist
Returns True, False if a service exists and it's not enabled
Returns True, True if a service exists and it's enabled
"""
try:
subprocess.check_call(
# the command return non-zero code if a service isn't enabled or
# it's not present
f"/usr/bin/systemctl is-enabled {service_name} &> /dev/null",
shell=True,
executable="/bin/bash",
)
return True, True
except (subprocess.CalledProcessError, FileNotFoundError,):
try:
p = subprocess.Popen(
[
"/usr/bin/systemctl",
"status",
service_name,
],
stderr=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
)
p.communicate()
# 0 - service is run
# 3 - service is stopped
# 4 - service doesn't exist
if p.returncode in (0, 3):
return True, False
else:
return False, False
except (subprocess.CalledProcessError, FileNotFoundError,):
return False, False
def service_is_enabled_and_present(service_name: str) -> Tuple[bool, bool]:
"""
Returns state of service (present and enabled)
:param service_name: name of a service
"""
cl_ver = get_cl_version()
if cl_ver and "cl6" in cl_ver:
is_present, is_enabled = _get_service_state_on_init_d_system(
service_name=service_name,
)
else:
is_present, is_enabled = _get_service_state_on_systemd_system(
service_name=service_name,
)
return is_present, is_enabled
def process_is_running(process_file_path: str, strict_match: bool,) -> bool:
"""
Check that a file in path is running.
You can get false-postive if parameter `strict_match` == False, process is
not running, but someone on server open file by path `process_file_path`
in an editor
:param process_file_path: path to a file which is run
:param strict_match: we use parameter `process_file_path` as full cmd line
with args for comparing if `strict_match` == True.
:return: True if it's running, False - is not,
"""
if not os.path.exists(process_file_path):
raise FileNotFoundError(
f'Process file in path "{process_file_path}" does not exist'
)
for process in psutil.process_iter(["cmdline"]):
try:
if (
not strict_match
and process_file_path in process.cmdline()
) or (
strict_match
and process_file_path == ' '.join(process.cmdline())
):
return True
except psutil.NoSuchProcess:
continue
return False
def get_weekday(dt: Union[datetime.datetime, datetime.date]) -> str:
"""
Getting string representation of weekday from datetime.datetime or datetime.date.
Returns shortened version of weekday from WEEK_DAYS.
"""
if not (isinstance(dt, (datetime.datetime, datetime.date))):
raise TypeError(
f"Require object of type datetime.datetime or datetime.date, but passed {type(dt)}"
)
return WEEK_DAYS[dt.weekday()]
def find_module_param_in_config(
config_path: str,
apache_module_name: str,
param_name: str,
default: int = None,
) -> Tuple[int, str]:
"""
Helper to parse httpd config for details about mpm module used
:param config_path: path for configuration file with modules
:param apache_module_name: expected mpm module. Can be `event`,
`worker`, `prefork`
:param param_name: name of parameter to find
:param default: default value for parameter, if there won't be record
:return: tuple with param value and text result of operation
Example of config file content:
<IfModule mpm_prefork_module>
.................
MaxRequestWorkers 450
</IfModule>
--
<IfModule mpm_worker_module>
.................
MaxRequestWorkers 300
</IfModule>
--
<IfModule mpm_event_module>
.................
MaxRequestWorkers 2048
</IfModule>
"""
if_module_line = f"<IfModule mpm_{apache_module_name}_module>"
section_lines = []
mpm_lines = get_file_lines(config_path)
is_section_found = False
for line in mpm_lines:
line = line.strip()
if line == if_module_line:
is_section_found = True
continue
if is_section_found and line == "</IfModule>":
# End of section
break
if is_section_found:
section_lines.append(line)
# 2. Find directive in found section
grep_result_list = list(
grep(
param_name,
multiple_search=True,
fixed_string=False,
data_from_file=section_lines,
)
)
mrw_list = [
directive.strip()
for directive in grep_result_list
if directive.strip().startswith(param_name)
]
# 3. Parse all lines with directive and find correct
# There is no custom setting for parameter
if not mrw_list and default is not None:
# Plesk case, when we can use defaults
return default, "OK"
elif not mrw_list and default is None:
# DA case, when we don't know about defaults
return (
0,
f"MaxRequestWorkers directive not found for "
f"mpm_{apache_module_name}_module module in {config_path}",
)
# There can be few records with MaxRequestWorkers, so we need
# to take the last one
parts = mrw_list[-1].split(" ")
max_request_workers = int(parts[-1])
return max_request_workers, "OK"
def get_kmodlve_module_version() -> str:
"""
Return kmodlve module's version.
Content of '/sys/module/kmodlve/version' looks like '2.0-30.el8'.
"""
kmodlve_module_file_path = "/sys/module/kmodlve/version"
if not os.path.isfile(kmodlve_module_file_path):
return ""
try:
with open(kmodlve_module_file_path, "r") as f:
module = f.read().strip()
module_version, _ = module.rsplit(".", 1)
except (OSError, IOError, ValueError):
return ""
return module_version
def proc_can_see_other_uid_and_hidepid_synced() -> bool:
"""
Find out if system has version of the kernel
(according to kmodlve module's version)
where fs.proc_can_see_other_uid and hidepid options are synchronized.
They are only synchronized if kmodlve module's version
is equal to version in synced_kmodlve_versions.
"""
synced_kmodlve_versions = {"cl8": "2.0-30"}
cl_version = get_cl_version()
if cl_version not in synced_kmodlve_versions:
return False
synced_version = synced_kmodlve_versions[cl_version]
current_version = get_kmodlve_module_version()
if current_version == synced_version:
return True
return False
def get_process_pid(pid_filename: str) -> Optional[int]:
"""
Detrmines working daemon process pid
:param pid_filename: PID filename
:return: PID from file or None if error (file not found, etc)
"""
try:
with open(pid_filename, "r") as pf:
pid = int(pf.read().strip())
# check that pid is still running
os.kill(pid, 0)
except (OSError, IOError, ValueError):
pid = None
return pid
def write_pid_file(pid_filename: str):
"""
Writes pid file
"""
with open(pid_filename, "w+") as pf:
pf.write(f"{os.getpid()}")
def remove_pid_file(pid_filename: str):
"""
Remove PID file
"""
try:
os.remove(pid_filename)
except (OSError, IOError):
pass
def demote(uid, gid):
"""
Set user's real uid and gid to specified ones.
Checking equality of real and effective uids is needed
because this function may be used by root
with effective uid dropped to user's uid.
In that case it is needed to set effective uid back to 0 first.
"""
def func():
real_uid = os.getuid()
eff_uid = os.geteuid()
if real_uid != eff_uid:
os.seteuid(real_uid)
os.setgid(gid)
os.setuid(uid)
return func
def get_mount_point(dirpath: str) -> str:
"""
Get mount point for dirpath directory from output of `df` utility
"""
if not os.path.isdir(dirpath):
raise OSError(f'Directory "{dirpath}" does not exist')
fs_info_cmd = ['/bin/df', '-h', dirpath]
fs_info = run_command(fs_info_cmd)
# example stdout:
# [root@localhost ~] # df -h /home
# Filesystem Size Used Avail Use% Mounted on
# /dev/vda1 10G 3.1G 7.0G 31% /
try:
mountpoint_info = fs_info.split('\n')[1]
mounted_on = mountpoint_info.split(' ')[-1]
except IndexError:
raise OSError(f'Utility "df" returned unexpected output:\n{fs_info}')
return mounted_on
Zerion Mini Shell 1.0