Saltstack troubleshooting: Difference between revisions
(→pip) |
(→Bugs) |
||
(One intermediate revision by the same user not shown) | |||
Line 76: | Line 76: | ||
</blockquote><!-- pip --> | </blockquote><!-- pip --> | ||
</blockquote><!-- Salt Modules won't Execute --> | </blockquote><!-- Salt Modules won't Execute --> | ||
= Bugs = | |||
<blockquote> | |||
== KeyError: 'config.option', pkg_resources.DistributionNotFound: The 'contextvars' distribution was not found and is required by salt == | |||
<blockquote> | |||
* See https://github.com/saltstack/salt/issues/65110 | |||
* And https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=272027 | |||
If you don't need <code>salt-ssh</code>, you can fix by removing <code>contextvars</code> from <code>/usr/local/lib/python3.9/site-packages/salt-3006.2-py3.9.egg-info/requires.txt</code> | |||
</blockquote><!-- KeyError: 'config.option' --> | |||
== zeromq.eventloop.future.CancelledError == | |||
<blockquote> | |||
{{ expand | |||
| full stacktrace | |||
| | |||
<pre> | |||
2024-02-16 06:35:56,635 [salt.cli.daemons :281 ][INFO ][2856] Setting up the Salt Minion "a0613" | |||
2024-02-16 06:35:56,739 [salt.cli.daemons :83 ][INFO ][2856] Starting up the Salt Minion | |||
2024-02-16 06:35:56,740 [salt.utils.event :1132][INFO ][2856] Starting pull socket on /var/run/salt/minion/minion_event_436c834f4b_pull.ipc | |||
2024-02-16 06:35:57,252 [salt.minion :1326][INFO ][2856] Creating minion process manager | |||
2024-02-16 06:36:57,264 [salt.minion :1128][ERROR ][2856] Minion unable to successfully connect to a Salt Master. | |||
2024-02-16 06:37:27,664 [salt.minion :1180][ERROR ][2856] Error while bringing up minion for multi-master. Is master at saltmaster responding? | |||
2024-02-16 06:39:30,584 [salt.minion :1180][ERROR ][2856] Error while bringing up minion for multi-master. Is master at saltmaster responding? | |||
2024-02-16 06:41:30,916 [salt.minion :1180][ERROR ][2856] Error while bringing up minion for multi-master. Is master at saltmaster responding? | |||
2024-02-16 06:42:46,054 [tornado.application:640 ][ERROR ][2856] Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x7fb38fc7af80>, <salt.ext.tornado.concurrent.Future object at 0x7fb38fca9570>) | |||
Traceback (most recent call last): | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 603, in _send_recv | |||
recv = yield self.socket.recv() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 1056, in run | |||
value = future.result() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 249, in result | |||
raise_exc_info(self._exc_info) | |||
File "<string>", line 4, in raise_exc_info | |||
zmq.eventloop.future.CancelledError | |||
During handling of the above exception, another exception occurred: | |||
Traceback (most recent call last): | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 605, in _send_recv | |||
future.set_exception(exc) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 294, in set_exception | |||
self.set_exc_info( | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 319, in set_exc_info | |||
self._set_done() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 333, in _set_done | |||
for cb in self._callbacks: | |||
TypeError: 'NoneType' object is not iterable | |||
During handling of the above exception, another exception occurred: | |||
Traceback (most recent call last): | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 606, in _run_callback | |||
ret = callback() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/stack_context.py", line 278, in null_wrapper | |||
return fn(*args, **kwargs) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 628, in _discard_future_result | |||
future.result() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 249, in result | |||
raise_exc_info(self._exc_info) | |||
File "<string>", line 4, in raise_exc_info | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 1064, in run | |||
yielded = self.gen.throw(*exc_info) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 612, in _send_recv | |||
future.set_exception(exc) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 294, in set_exception | |||
self.set_exc_info( | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 319, in set_exc_info | |||
self._set_done() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 333, in _set_done | |||
for cb in self._callbacks: | |||
TypeError: 'NoneType' object is not iterable | |||
2024-02-16 06:42:46,057 [tornado.application:640 ][ERROR ][2856] Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x7fb38fd26560>, <salt.ext.tornado.concurrent.Future object at 0x7fb38fd18100>) | |||
Traceback (most recent call last): | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 601, in _send_recv | |||
yield self.socket.send(message) | |||
AttributeError: 'NoneType' object has no attribute 'send' | |||
During handling of the above exception, another exception occurred: | |||
Traceback (most recent call last): | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 606, in _run_callback | |||
ret = callback() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/stack_context.py", line 278, in null_wrapper | |||
return fn(*args, **kwargs) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 628, in _discard_future_result | |||
future.result() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 249, in result | |||
raise_exc_info(self._exc_info) | |||
File "<string>", line 4, in raise_exc_info | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 1070, in run | |||
yielded = self.gen.send(value) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 612, in _send_recv | |||
future.set_exception(exc) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 294, in set_exception | |||
self.set_exc_info( | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 319, in set_exc_info | |||
self._set_done() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 333, in _set_done | |||
for cb in self._callbacks: | |||
TypeError: 'NoneType' object is not iterable | |||
2024-02-16 06:42:46,057 [salt.minion :1180][ERROR ][2856] Error while bringing up minion for multi-master. Is master at saltmine.atlas.local responding? | |||
2024-02-16 06:42:46,058 [tornado.application:640 ][ERROR ][2856] Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x7fb38fd26a70>, <salt.ext.tornado.concurrent.Future object at 0x7fb38fd18b80>) | |||
Traceback (most recent call last): | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 601, in _send_recv | |||
yield self.socket.send(message) | |||
AttributeError: 'NoneType' object has no attribute 'send' | |||
During handling of the above exception, another exception occurred: | |||
Traceback (most recent call last): | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 606, in _run_callback | |||
ret = callback() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/stack_context.py", line 278, in null_wrapper | |||
return fn(*args, **kwargs) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 628, in _discard_future_result | |||
future.result() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 249, in result | |||
raise_exc_info(self._exc_info) | |||
File "<string>", line 4, in raise_exc_info | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 1070, in run | |||
yielded = self.gen.send(value) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 612, in _send_recv | |||
future.set_exception(exc) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 294, in set_exception | |||
self.set_exc_info( | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 319, in set_exc_info | |||
self._set_done() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 333, in _set_done | |||
for cb in self._callbacks: | |||
TypeError: 'NoneType' object is not iterable | |||
2024-02-16 06:42:46,059 [tornado.application:640 ][ERROR ][2856] Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x7fb38fd27250>, <salt.ext.tornado.concurrent.Future object at 0x7fb38fd1bf40>) | |||
Traceback (most recent call last): | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 601, in _send_recv | |||
yield self.socket.send(message) | |||
AttributeError: 'NoneType' object has no attribute 'send' | |||
During handling of the above exception, another exception occurred: | |||
Traceback (most recent call last): | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 606, in _run_callback | |||
ret = callback() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/stack_context.py", line 278, in null_wrapper | |||
return fn(*args, **kwargs) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 628, in _discard_future_result | |||
future.result() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 249, in result | |||
raise_exc_info(self._exc_info) | |||
File "<string>", line 4, in raise_exc_info | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 1070, in run | |||
yielded = self.gen.send(value) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 612, in _send_recv | |||
future.set_exception(exc) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 294, in set_exception | |||
self.set_exc_info( | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 319, in set_exc_info | |||
self._set_done() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 333, in _set_done | |||
for cb in self._callbacks: | |||
TypeError: 'NoneType' object is not iterable | |||
2024-02-16 06:42:46,059 [tornado.application:640 ][ERROR ][2856] Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x7fb38fd27a30>, <salt.ext.tornado.concurrent.Future object at 0x7fb38fd3c910>) | |||
Traceback (most recent call last): | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 601, in _send_recv | |||
yield self.socket.send(message) | |||
AttributeError: 'NoneType' object has no attribute 'send' | |||
During handling of the above exception, another exception occurred: | |||
Traceback (most recent call last): | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 606, in _run_callback | |||
ret = callback() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/stack_context.py", line 278, in null_wrapper | |||
return fn(*args, **kwargs) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 628, in _discard_future_result | |||
future.result() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 249, in result | |||
raise_exc_info(self._exc_info) | |||
File "<string>", line 4, in raise_exc_info | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 1070, in run | |||
yielded = self.gen.send(value) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 612, in _send_recv | |||
future.set_exception(exc) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 294, in set_exception | |||
self.set_exc_info( | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 319, in set_exc_info | |||
self._set_done() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 333, in _set_done | |||
for cb in self._callbacks: | |||
TypeError: 'NoneType' object is not iterable | |||
2024-02-16 06:42:46,060 [tornado.application:640 ][ERROR ][2856] Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x7fb37e70c280>, <salt.ext.tornado.concurrent.Future object at 0x7fb38fd3d2a0>) | |||
Traceback (most recent call last): | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 601, in _send_recv | |||
yield self.socket.send(message) | |||
AttributeError: 'NoneType' object has no attribute 'send' | |||
During handling of the above exception, another exception occurred: | |||
Traceback (most recent call last): | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 606, in _run_callback | |||
ret = callback() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/stack_context.py", line 278, in null_wrapper | |||
return fn(*args, **kwargs) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 628, in _discard_future_result | |||
future.result() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 249, in result | |||
raise_exc_info(self._exc_info) | |||
File "<string>", line 4, in raise_exc_info | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 1070, in run | |||
yielded = self.gen.send(value) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 612, in _send_recv | |||
future.set_exception(exc) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 294, in set_exception | |||
self.set_exc_info( | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 319, in set_exc_info | |||
self._set_done() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 333, in _set_done | |||
for cb in self._callbacks: | |||
TypeError: 'NoneType' object is not iterable | |||
2024-02-16 06:43:36,108 [salt.minion :1200][CRITICAL][2856] Unexpected error while connecting to saltmine.atlas.local | |||
Traceback (most recent call last): | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/minion.py", line 1174, in _connect_minion | |||
yield minion.connect_master(failed=failed) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 1056, in run | |||
value = future.result() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 249, in result | |||
raise_exc_info(self._exc_info) | |||
File "<string>", line 4, in raise_exc_info | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 309, in wrapper | |||
yielded = next(result) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/minion.py", line 1401, in connect_master | |||
self.pub_channel.on_recv(None) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/channel/client.py", line 467, in on_recv | |||
return self.transport.on_recv(None) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 284, in on_recv | |||
return self.stream.on_recv(callback) | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/zmq/eventloop/zmqstream.py", line 214, in on_recv | |||
self._check_closed() | |||
File "/opt/saltstack/salt/lib/python3.10/site-packages/zmq/eventloop/zmqstream.py", line 628, in _check_closed | |||
raise OSError("Stream is closed") | |||
</pre> | |||
}} | |||
Fix: | |||
<syntaxhighlight lang="yaml"> | |||
# /usr/local/etc/salt/minion | |||
request_channel_timeout: 120 # defaults 30s | |||
</syntaxhighlight> | |||
</blockquote><!-- zeromq.eventloop.future.CancelledError --> | |||
</blockquote><!-- Bugs --> |
Latest revision as of 18:19, 19 February 2024
Invalid Jinja
Check the minion's
/var/log/salt/minion
to get the full python stacktrace.
Salt Modules won't Execute
Here there be dragons.
If you're lucky, the salt modules documentation will stipulate a requirement. For example, see the note on salt.modules.gpg.
In most cases, a problem can be pinned down by reading stacktraces in/var/log/salt/minion
.Here are some modules that I bang my head against frequently.
gpg
- install gpg
- install python-gpg to saltstack's python interpreter
pip
Most of my problems here come from:
- salt using the global python installation rather than a virtualenv
- my package manager installing a newer package than salt supports.
Unfortunately this is a moving target, and pinning packages becomes a bit of a headache.
(I did try salt in a virtualenv, but gave after hours debugging issues with salt's protocol decoding)Verify using logs:
sudo less /var/log/salt/minion
.Failed to import states pip_state, this is due most likely to a syntax error:Sample Resolutions
Archlinux, Package Downgrade
# uninstall pkg while still required by salt pacman -Rsd --nodeps python-pyzmq # find/download old version of package xdg-open https://archive.archlinux.org/packages # install old package pacman -U ~/Downloads/python-pyzmq-19.0.1-4-x86_64.pkg.tar.zst # pin the package version so it doesn't get updated # /etc/pacman.conf IgnorePkg = python-pyzmqIf that doesn't work, confirm that the same python install is being used.
You ma/var/log/salt/miniony need to downgrade saltstack, or pip install ${PKG} into saltstack's python interpreter.find /usr/lib -type d -name '*zmq*' find /usr/lib -type d -name '*salt*'/usr/bin/python -m pip install 'pyzmq<=20.0.0'Archlinux, Deps
pkg_resources
, required bypip.installed
, now requirespsutil
.pacman -S python-psutil
Bugs
KeyError: 'config.option', pkg_resources.DistributionNotFound: The 'contextvars' distribution was not found and is required by salt
- See https://github.com/saltstack/salt/issues/65110
- And https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=272027
If you don't need
salt-ssh
, you can fix by removingcontextvars
from/usr/local/lib/python3.9/site-packages/salt-3006.2-py3.9.egg-info/requires.txt
zeromq.eventloop.future.CancelledError
full stacktrace
2024-02-16 06:35:56,635 [salt.cli.daemons :281 ][INFO ][2856] Setting up the Salt Minion "a0613" 2024-02-16 06:35:56,739 [salt.cli.daemons :83 ][INFO ][2856] Starting up the Salt Minion 2024-02-16 06:35:56,740 [salt.utils.event :1132][INFO ][2856] Starting pull socket on /var/run/salt/minion/minion_event_436c834f4b_pull.ipc 2024-02-16 06:35:57,252 [salt.minion :1326][INFO ][2856] Creating minion process manager 2024-02-16 06:36:57,264 [salt.minion :1128][ERROR ][2856] Minion unable to successfully connect to a Salt Master. 2024-02-16 06:37:27,664 [salt.minion :1180][ERROR ][2856] Error while bringing up minion for multi-master. Is master at saltmaster responding? 2024-02-16 06:39:30,584 [salt.minion :1180][ERROR ][2856] Error while bringing up minion for multi-master. Is master at saltmaster responding? 2024-02-16 06:41:30,916 [salt.minion :1180][ERROR ][2856] Error while bringing up minion for multi-master. Is master at saltmaster responding? 2024-02-16 06:42:46,054 [tornado.application:640 ][ERROR ][2856] Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x7fb38fc7af80>, <salt.ext.tornado.concurrent.Future object at 0x7fb38fca9570>) Traceback (most recent call last): File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 603, in _send_recv recv = yield self.socket.recv() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 1056, in run value = future.result() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 249, in result raise_exc_info(self._exc_info) File "<string>", line 4, in raise_exc_info zmq.eventloop.future.CancelledError During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 605, in _send_recv future.set_exception(exc) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 294, in set_exception self.set_exc_info( File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 319, in set_exc_info self._set_done() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 333, in _set_done for cb in self._callbacks: TypeError: 'NoneType' object is not iterable During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 606, in _run_callback ret = callback() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/stack_context.py", line 278, in null_wrapper return fn(*args, **kwargs) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 628, in _discard_future_result future.result() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 249, in result raise_exc_info(self._exc_info) File "<string>", line 4, in raise_exc_info File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 1064, in run yielded = self.gen.throw(*exc_info) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 612, in _send_recv future.set_exception(exc) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 294, in set_exception self.set_exc_info( File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 319, in set_exc_info self._set_done() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 333, in _set_done for cb in self._callbacks: TypeError: 'NoneType' object is not iterable 2024-02-16 06:42:46,057 [tornado.application:640 ][ERROR ][2856] Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x7fb38fd26560>, <salt.ext.tornado.concurrent.Future object at 0x7fb38fd18100>) Traceback (most recent call last): File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 601, in _send_recv yield self.socket.send(message) AttributeError: 'NoneType' object has no attribute 'send' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 606, in _run_callback ret = callback() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/stack_context.py", line 278, in null_wrapper return fn(*args, **kwargs) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 628, in _discard_future_result future.result() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 249, in result raise_exc_info(self._exc_info) File "<string>", line 4, in raise_exc_info File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 1070, in run yielded = self.gen.send(value) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 612, in _send_recv future.set_exception(exc) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 294, in set_exception self.set_exc_info( File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 319, in set_exc_info self._set_done() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 333, in _set_done for cb in self._callbacks: TypeError: 'NoneType' object is not iterable 2024-02-16 06:42:46,057 [salt.minion :1180][ERROR ][2856] Error while bringing up minion for multi-master. Is master at saltmine.atlas.local responding? 2024-02-16 06:42:46,058 [tornado.application:640 ][ERROR ][2856] Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x7fb38fd26a70>, <salt.ext.tornado.concurrent.Future object at 0x7fb38fd18b80>) Traceback (most recent call last): File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 601, in _send_recv yield self.socket.send(message) AttributeError: 'NoneType' object has no attribute 'send' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 606, in _run_callback ret = callback() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/stack_context.py", line 278, in null_wrapper return fn(*args, **kwargs) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 628, in _discard_future_result future.result() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 249, in result raise_exc_info(self._exc_info) File "<string>", line 4, in raise_exc_info File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 1070, in run yielded = self.gen.send(value) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 612, in _send_recv future.set_exception(exc) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 294, in set_exception self.set_exc_info( File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 319, in set_exc_info self._set_done() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 333, in _set_done for cb in self._callbacks: TypeError: 'NoneType' object is not iterable 2024-02-16 06:42:46,059 [tornado.application:640 ][ERROR ][2856] Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x7fb38fd27250>, <salt.ext.tornado.concurrent.Future object at 0x7fb38fd1bf40>) Traceback (most recent call last): File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 601, in _send_recv yield self.socket.send(message) AttributeError: 'NoneType' object has no attribute 'send' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 606, in _run_callback ret = callback() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/stack_context.py", line 278, in null_wrapper return fn(*args, **kwargs) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 628, in _discard_future_result future.result() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 249, in result raise_exc_info(self._exc_info) File "<string>", line 4, in raise_exc_info File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 1070, in run yielded = self.gen.send(value) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 612, in _send_recv future.set_exception(exc) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 294, in set_exception self.set_exc_info( File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 319, in set_exc_info self._set_done() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 333, in _set_done for cb in self._callbacks: TypeError: 'NoneType' object is not iterable 2024-02-16 06:42:46,059 [tornado.application:640 ][ERROR ][2856] Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x7fb38fd27a30>, <salt.ext.tornado.concurrent.Future object at 0x7fb38fd3c910>) Traceback (most recent call last): File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 601, in _send_recv yield self.socket.send(message) AttributeError: 'NoneType' object has no attribute 'send' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 606, in _run_callback ret = callback() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/stack_context.py", line 278, in null_wrapper return fn(*args, **kwargs) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 628, in _discard_future_result future.result() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 249, in result raise_exc_info(self._exc_info) File "<string>", line 4, in raise_exc_info File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 1070, in run yielded = self.gen.send(value) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 612, in _send_recv future.set_exception(exc) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 294, in set_exception self.set_exc_info( File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 319, in set_exc_info self._set_done() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 333, in _set_done for cb in self._callbacks: TypeError: 'NoneType' object is not iterable 2024-02-16 06:42:46,060 [tornado.application:640 ][ERROR ][2856] Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x7fb37e70c280>, <salt.ext.tornado.concurrent.Future object at 0x7fb38fd3d2a0>) Traceback (most recent call last): File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 601, in _send_recv yield self.socket.send(message) AttributeError: 'NoneType' object has no attribute 'send' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 606, in _run_callback ret = callback() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/stack_context.py", line 278, in null_wrapper return fn(*args, **kwargs) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/ioloop.py", line 628, in _discard_future_result future.result() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 249, in result raise_exc_info(self._exc_info) File "<string>", line 4, in raise_exc_info File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 1070, in run yielded = self.gen.send(value) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 612, in _send_recv future.set_exception(exc) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 294, in set_exception self.set_exc_info( File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 319, in set_exc_info self._set_done() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 333, in _set_done for cb in self._callbacks: TypeError: 'NoneType' object is not iterable 2024-02-16 06:43:36,108 [salt.minion :1200][CRITICAL][2856] Unexpected error while connecting to saltmine.atlas.local Traceback (most recent call last): File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/minion.py", line 1174, in _connect_minion yield minion.connect_master(failed=failed) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 1056, in run value = future.result() File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/concurrent.py", line 249, in result raise_exc_info(self._exc_info) File "<string>", line 4, in raise_exc_info File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/ext/tornado/gen.py", line 309, in wrapper yielded = next(result) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/minion.py", line 1401, in connect_master self.pub_channel.on_recv(None) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/channel/client.py", line 467, in on_recv return self.transport.on_recv(None) File "/opt/saltstack/salt/lib/python3.10/site-packages/salt/transport/zeromq.py", line 284, in on_recv return self.stream.on_recv(callback) File "/opt/saltstack/salt/lib/python3.10/site-packages/zmq/eventloop/zmqstream.py", line 214, in on_recv self._check_closed() File "/opt/saltstack/salt/lib/python3.10/site-packages/zmq/eventloop/zmqstream.py", line 628, in _check_closed raise OSError("Stream is closed")
Fix:
# /usr/local/etc/salt/minion request_channel_timeout: 120 # defaults 30s