1
# Copyright 2001-2014 by Vinay Sajip. All Rights Reserved.
3
# Permission to use, copy, modify, and distribute this software and its
4
# documentation for any purpose and without fee is hereby granted,
5
# provided that the above copyright notice appear in all copies and that
6
# both that copyright notice and this permission notice appear in
7
# supporting documentation, and that the name of Vinay Sajip
8
# not be used in advertising or publicity pertaining to distribution
9
# of the software without specific, written prior permission.
10
# VINAY SAJIP DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
11
# ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
12
# VINAY SAJIP BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR
13
# ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
14
# IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
15
# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18
Configuration functions for the logging package for Python. The core package
19
is based on PEP 282 and comments thereto in comp.lang.python, and influenced
20
by Apache's log4j system.
22
Copyright (C) 2001-2014 Vinay Sajip. All Rights Reserved.
24
To use, simply 'import logging' and log away!
31
import logging.handlers
46
from SocketServer import ThreadingTCPServer, StreamRequestHandler
49
DEFAULT_LOGGING_CONFIG_PORT = 9030
51
RESET_ERROR = errno.ECONNRESET
54
# The following code implements a socket listener for on-the-fly
55
# reconfiguration of logging.
57
# _listener holds the server object doing the listening
60
def fileConfig(fname, defaults=None, disable_existing_loggers=True):
62
Read the logging configuration from a ConfigParser-format file.
64
This can be called several times from an application, allowing an end user
65
the ability to select from various pre-canned configurations (if the
66
developer provides a mechanism to present the choices and load the chosen
71
cp = ConfigParser.ConfigParser(defaults)
72
if hasattr(fname, 'readline'):
77
formatters = _create_formatters(cp)
80
logging._acquireLock()
82
logging._handlers.clear()
83
del logging._handlerList[:]
84
# Handlers add themselves to logging._handlers
85
handlers = _install_handlers(cp, formatters)
86
_install_loggers(cp, handlers, disable_existing_loggers)
88
logging._releaseLock()
92
"""Resolve a dotted name to a global object."""
93
name = name.split('.')
95
found = __import__(used)
99
found = getattr(found, n)
100
except AttributeError:
102
found = getattr(found, n)
105
def _strip_spaces(alist):
106
return map(lambda x: x.strip(), alist)
109
return s if isinstance(s, str) else s.encode('utf-8')
111
def _create_formatters(cp):
112
"""Create and return formatters"""
113
flist = cp.get("formatters", "keys")
116
flist = flist.split(",")
117
flist = _strip_spaces(flist)
120
sectname = "formatter_%s" % form
121
opts = cp.options(sectname)
123
fs = cp.get(sectname, "format", 1)
126
if "datefmt" in opts:
127
dfs = cp.get(sectname, "datefmt", 1)
130
c = logging.Formatter
132
class_name = cp.get(sectname, "class")
134
c = _resolve(class_name)
140
def _install_handlers(cp, formatters):
141
"""Install and return handlers"""
142
hlist = cp.get("handlers", "keys")
145
hlist = hlist.split(",")
146
hlist = _strip_spaces(hlist)
148
fixups = [] #for inter-handler references
150
sectname = "handler_%s" % hand
151
klass = cp.get(sectname, "class")
152
opts = cp.options(sectname)
153
if "formatter" in opts:
154
fmt = cp.get(sectname, "formatter")
158
klass = eval(klass, vars(logging))
159
except (AttributeError, NameError):
160
klass = _resolve(klass)
161
args = cp.get(sectname, "args")
162
args = eval(args, vars(logging))
165
level = cp.get(sectname, "level")
166
h.setLevel(logging._levelNames[level])
168
h.setFormatter(formatters[fmt])
169
if issubclass(klass, logging.handlers.MemoryHandler):
171
target = cp.get(sectname,"target")
174
if len(target): #the target handler may not be loaded yet, so keep for later...
175
fixups.append((h, target))
177
#now all handlers are loaded, fixup inter-handler references...
179
h.setTarget(handlers[t])
183
def _install_loggers(cp, handlers, disable_existing_loggers):
184
"""Create and install loggers"""
186
# configure the root first
187
llist = cp.get("loggers", "keys")
188
llist = llist.split(",")
189
llist = list(map(lambda x: x.strip(), llist))
191
sectname = "logger_root"
194
opts = cp.options(sectname)
196
level = cp.get(sectname, "level")
197
log.setLevel(logging._levelNames[level])
198
for h in root.handlers[:]:
199
root.removeHandler(h)
200
hlist = cp.get(sectname, "handlers")
202
hlist = hlist.split(",")
203
hlist = _strip_spaces(hlist)
205
log.addHandler(handlers[hand])
207
#and now the others...
208
#we don't want to lose the existing loggers,
209
#since other threads may have pointers to them.
210
#existing is set to contain all existing loggers,
211
#and as we go through the new configuration we
212
#remove any which are configured. At the end,
213
#what's left in existing is the set of loggers
214
#which were in the previous configuration but
215
#which are not in the new configuration.
216
existing = list(root.manager.loggerDict.keys())
217
#The list needs to be sorted so that we can
218
#avoid disabling child loggers of explicitly
219
#named loggers. With a sorted list it is easier
220
#to find the child loggers.
222
#We'll keep the list of existing loggers
223
#which are children of named loggers here...
225
#now set up the new ones...
227
sectname = "logger_%s" % log
228
qn = cp.get(sectname, "qualname")
229
opts = cp.options(sectname)
230
if "propagate" in opts:
231
propagate = cp.getint(sectname, "propagate")
234
logger = logging.getLogger(qn)
236
i = existing.index(qn) + 1 # start with the entry after qn
238
pflen = len(prefixed)
239
num_existing = len(existing)
240
while i < num_existing:
241
if existing[i][:pflen] == prefixed:
242
child_loggers.append(existing[i])
246
level = cp.get(sectname, "level")
247
logger.setLevel(logging._levelNames[level])
248
for h in logger.handlers[:]:
249
logger.removeHandler(h)
250
logger.propagate = propagate
252
hlist = cp.get(sectname, "handlers")
254
hlist = hlist.split(",")
255
hlist = _strip_spaces(hlist)
257
logger.addHandler(handlers[hand])
259
#Disable any old loggers. There's no point deleting
260
#them as other threads may continue to hold references
261
#and by disabling them, you stop them doing any logging.
262
#However, don't disable children of named loggers, as that's
263
#probably not what was intended by the user.
265
logger = root.manager.loggerDict[log]
266
if log in child_loggers:
267
logger.level = logging.NOTSET
271
logger.disabled = disable_existing_loggers
275
IDENTIFIER = re.compile('^[a-z_][a-z0-9_]*$', re.I)
279
m = IDENTIFIER.match(s)
281
raise ValueError('Not a valid Python identifier: %r' % s)
285
class ConvertingMixin(object):
286
"""For ConvertingXXX's, this mixin class provides common functions"""
288
def convert_with_key(self, key, value, replace=True):
289
result = self.configurator.convert(value)
290
#If the converted value is different, save for next time
291
if value is not result:
294
if type(result) in (ConvertingDict, ConvertingList,
300
def convert(self, value):
301
result = self.configurator.convert(value)
302
if value is not result:
303
if type(result) in (ConvertingDict, ConvertingList,
309
# The ConvertingXXX classes are wrappers around standard Python containers,
310
# and they serve to convert any suitable values in the container. The
311
# conversion converts base dicts, lists and tuples to their wrapped
312
# equivalents, whereas strings which match a conversion format are converted
315
# Each wrapper should have a configurator attribute holding the actual
316
# configurator to use for conversion.
318
class ConvertingDict(dict, ConvertingMixin):
319
"""A converting dictionary wrapper."""
321
def __getitem__(self, key):
322
value = dict.__getitem__(self, key)
323
return self.convert_with_key(key, value)
325
def get(self, key, default=None):
326
value = dict.get(self, key, default)
327
return self.convert_with_key(key, value)
329
def pop(self, key, default=None):
330
value = dict.pop(self, key, default)
331
return self.convert_with_key(key, value, replace=False)
333
class ConvertingList(list, ConvertingMixin):
334
"""A converting list wrapper."""
335
def __getitem__(self, key):
336
value = list.__getitem__(self, key)
337
return self.convert_with_key(key, value)
339
def pop(self, idx=-1):
340
value = list.pop(self, idx)
341
return self.convert(value)
343
class ConvertingTuple(tuple, ConvertingMixin):
344
"""A converting tuple wrapper."""
345
def __getitem__(self, key):
346
value = tuple.__getitem__(self, key)
347
# Can't replace a tuple entry.
348
return self.convert_with_key(key, value, replace=False)
350
class BaseConfigurator(object):
352
The configurator base class which defines some useful defaults.
355
CONVERT_PATTERN = re.compile(r'^(?P<prefix>[a-z]+)://(?P<suffix>.*)$')
357
WORD_PATTERN = re.compile(r'^\s*(\w+)\s*')
358
DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*')
359
INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*')
360
DIGIT_PATTERN = re.compile(r'^\d+$')
363
'ext' : 'ext_convert',
364
'cfg' : 'cfg_convert',
367
# We might want to use a different one, e.g. importlib
368
importer = __import__
370
def __init__(self, config):
371
self.config = ConvertingDict(config)
372
self.config.configurator = self
373
# Issue 12718: winpdb replaces __import__ with a Python function, which
374
# ends up being treated as a bound method. To avoid problems, we
375
# set the importer on the instance, but leave it defined in the class
376
# so existing code doesn't break
377
if type(__import__) == types.FunctionType:
378
self.importer = __import__
380
def resolve(self, s):
382
Resolve strings to objects using standard import and attribute
388
found = self.importer(used)
392
found = getattr(found, frag)
393
except AttributeError:
395
found = getattr(found, frag)
398
e, tb = sys.exc_info()[1:]
399
v = ValueError('Cannot resolve %r: %s' % (s, e))
400
v.__cause__, v.__traceback__ = e, tb
403
def ext_convert(self, value):
404
"""Default converter for the ext:// protocol."""
405
return self.resolve(value)
407
def cfg_convert(self, value):
408
"""Default converter for the cfg:// protocol."""
410
m = self.WORD_PATTERN.match(rest)
412
raise ValueError("Unable to convert %r" % value)
414
rest = rest[m.end():]
415
d = self.config[m.groups()[0]]
418
m = self.DOT_PATTERN.match(rest)
422
m = self.INDEX_PATTERN.match(rest)
425
if not self.DIGIT_PATTERN.match(idx):
429
n = int(idx) # try as number first (most likely)
434
rest = rest[m.end():]
436
raise ValueError('Unable to convert '
437
'%r at %r' % (value, rest))
438
#rest should be empty
441
def convert(self, value):
443
Convert values to an appropriate type. dicts, lists and tuples are
444
replaced by their converting alternatives. Strings are checked to
445
see if they have a conversion format and are converted if they do.
447
if not isinstance(value, ConvertingDict) and isinstance(value, dict):
448
value = ConvertingDict(value)
449
value.configurator = self
450
elif not isinstance(value, ConvertingList) and isinstance(value, list):
451
value = ConvertingList(value)
452
value.configurator = self
453
elif not isinstance(value, ConvertingTuple) and\
454
isinstance(value, tuple):
455
value = ConvertingTuple(value)
456
value.configurator = self
457
elif isinstance(value, basestring): # str for py3k
458
m = self.CONVERT_PATTERN.match(value)
462
converter = self.value_converters.get(prefix, None)
465
converter = getattr(self, converter)
466
value = converter(suffix)
469
def configure_custom(self, config):
470
"""Configure an object with a user-supplied factory."""
472
if not hasattr(c, '__call__') and hasattr(types, 'ClassType') and type(c) != types.ClassType:
474
props = config.pop('.', None)
475
# Check for valid identifiers
476
kwargs = dict([(k, config[k]) for k in config if valid_ident(k)])
479
for name, value in props.items():
480
setattr(result, name, value)
483
def as_tuple(self, value):
484
"""Utility function which converts lists to tuples."""
485
if isinstance(value, list):
489
class DictConfigurator(BaseConfigurator):
491
Configure logging using a dictionary-like object to describe the
496
"""Do the configuration."""
499
if 'version' not in config:
500
raise ValueError("dictionary doesn't specify a version")
501
if config['version'] != 1:
502
raise ValueError("Unsupported version: %s" % config['version'])
503
incremental = config.pop('incremental', False)
505
logging._acquireLock()
508
handlers = config.get('handlers', EMPTY_DICT)
509
for name in handlers:
510
if name not in logging._handlers:
511
raise ValueError('No handler found with '
515
handler = logging._handlers[name]
516
handler_config = handlers[name]
517
level = handler_config.get('level', None)
519
handler.setLevel(logging._checkLevel(level))
520
except StandardError as e:
521
raise ValueError('Unable to configure handler '
522
'%r: %s' % (name, e))
523
loggers = config.get('loggers', EMPTY_DICT)
526
self.configure_logger(name, loggers[name], True)
527
except StandardError as e:
528
raise ValueError('Unable to configure logger '
529
'%r: %s' % (name, e))
530
root = config.get('root', None)
533
self.configure_root(root, True)
534
except StandardError as e:
535
raise ValueError('Unable to configure root '
538
disable_existing = config.pop('disable_existing_loggers', True)
540
logging._handlers.clear()
541
del logging._handlerList[:]
543
# Do formatters first - they don't refer to anything else
544
formatters = config.get('formatters', EMPTY_DICT)
545
for name in formatters:
547
formatters[name] = self.configure_formatter(
549
except StandardError as e:
550
raise ValueError('Unable to configure '
551
'formatter %r: %s' % (name, e))
552
# Next, do filters - they don't refer to anything else, either
553
filters = config.get('filters', EMPTY_DICT)
556
filters[name] = self.configure_filter(filters[name])
557
except StandardError as e:
558
raise ValueError('Unable to configure '
559
'filter %r: %s' % (name, e))
561
# Next, do handlers - they refer to formatters and filters
562
# As handlers can refer to other handlers, sort the keys
563
# to allow a deterministic order of configuration
564
handlers = config.get('handlers', EMPTY_DICT)
566
for name in sorted(handlers):
568
handler = self.configure_handler(handlers[name])
570
handlers[name] = handler
571
except StandardError as e:
572
if 'target not configured yet' in str(e):
573
deferred.append(name)
575
raise ValueError('Unable to configure handler '
576
'%r: %s' % (name, e))
578
# Now do any that were deferred
579
for name in deferred:
581
handler = self.configure_handler(handlers[name])
583
handlers[name] = handler
584
except StandardError as e:
585
raise ValueError('Unable to configure handler '
586
'%r: %s' % (name, e))
588
# Next, do loggers - they refer to handlers and filters
590
#we don't want to lose the existing loggers,
591
#since other threads may have pointers to them.
592
#existing is set to contain all existing loggers,
593
#and as we go through the new configuration we
594
#remove any which are configured. At the end,
595
#what's left in existing is the set of loggers
596
#which were in the previous configuration but
597
#which are not in the new configuration.
599
existing = root.manager.loggerDict.keys()
600
#The list needs to be sorted so that we can
601
#avoid disabling child loggers of explicitly
602
#named loggers. With a sorted list it is easier
603
#to find the child loggers.
605
#We'll keep the list of existing loggers
606
#which are children of named loggers here...
608
#now set up the new ones...
609
loggers = config.get('loggers', EMPTY_DICT)
611
name = _encoded(name)
613
i = existing.index(name)
614
prefixed = name + "."
615
pflen = len(prefixed)
616
num_existing = len(existing)
617
i = i + 1 # look at the entry after name
618
while (i < num_existing) and\
619
(existing[i][:pflen] == prefixed):
620
child_loggers.append(existing[i])
622
existing.remove(name)
624
self.configure_logger(name, loggers[name])
625
except StandardError as e:
626
raise ValueError('Unable to configure logger '
627
'%r: %s' % (name, e))
629
#Disable any old loggers. There's no point deleting
630
#them as other threads may continue to hold references
631
#and by disabling them, you stop them doing any logging.
632
#However, don't disable children of named loggers, as that's
633
#probably not what was intended by the user.
635
logger = root.manager.loggerDict[log]
636
if log in child_loggers:
637
logger.level = logging.NOTSET
639
logger.propagate = True
640
elif disable_existing:
641
logger.disabled = True
643
# And finally, do the root logger
644
root = config.get('root', None)
647
self.configure_root(root)
648
except StandardError as e:
649
raise ValueError('Unable to configure root '
652
logging._releaseLock()
654
def configure_formatter(self, config):
655
"""Configure a formatter from a dictionary."""
657
factory = config['()'] # for use in exception handler
659
result = self.configure_custom(config)
660
except TypeError as te:
661
if "'format'" not in str(te):
663
#Name of parameter changed from fmt to format.
664
#Retry with old name.
665
#This is so that code can be used with older Python versions
667
config['fmt'] = config.pop('format')
668
config['()'] = factory
669
result = self.configure_custom(config)
671
fmt = config.get('format', None)
672
dfmt = config.get('datefmt', None)
673
result = logging.Formatter(fmt, dfmt)
676
def configure_filter(self, config):
677
"""Configure a filter from a dictionary."""
679
result = self.configure_custom(config)
681
name = config.get('name', '')
682
result = logging.Filter(name)
685
def add_filters(self, filterer, filters):
686
"""Add filters to a filterer from a list of names."""
689
filterer.addFilter(self.config['filters'][f])
690
except StandardError as e:
691
raise ValueError('Unable to add filter %r: %s' % (f, e))
693
def configure_handler(self, config):
694
"""Configure a handler from a dictionary."""
695
formatter = config.pop('formatter', None)
698
formatter = self.config['formatters'][formatter]
699
except StandardError as e:
700
raise ValueError('Unable to set formatter '
701
'%r: %s' % (formatter, e))
702
level = config.pop('level', None)
703
filters = config.pop('filters', None)
706
if not hasattr(c, '__call__') and hasattr(types, 'ClassType') and type(c) != types.ClassType:
710
cname = config.pop('class')
711
klass = self.resolve(cname)
712
#Special case for handler which refers to another handler
713
if issubclass(klass, logging.handlers.MemoryHandler) and\
716
th = self.config['handlers'][config['target']]
717
if not isinstance(th, logging.Handler):
718
config['class'] = cname # restore for deferred configuration
719
raise StandardError('target not configured yet')
720
config['target'] = th
721
except StandardError as e:
722
raise ValueError('Unable to set target handler '
723
'%r: %s' % (config['target'], e))
724
elif issubclass(klass, logging.handlers.SMTPHandler) and\
725
'mailhost' in config:
726
config['mailhost'] = self.as_tuple(config['mailhost'])
727
elif issubclass(klass, logging.handlers.SysLogHandler) and\
729
config['address'] = self.as_tuple(config['address'])
731
kwargs = dict([(k, config[k]) for k in config if valid_ident(k)])
733
result = factory(**kwargs)
734
except TypeError as te:
735
if "'stream'" not in str(te):
737
#The argument name changed from strm to stream
738
#Retry with old name.
739
#This is so that code can be used with older Python versions
741
kwargs['strm'] = kwargs.pop('stream')
742
result = factory(**kwargs)
744
result.setFormatter(formatter)
745
if level is not None:
746
result.setLevel(logging._checkLevel(level))
748
self.add_filters(result, filters)
751
def add_handlers(self, logger, handlers):
752
"""Add handlers to a logger from a list of names."""
755
logger.addHandler(self.config['handlers'][h])
756
except StandardError as e:
757
raise ValueError('Unable to add handler %r: %s' % (h, e))
759
def common_logger_config(self, logger, config, incremental=False):
761
Perform configuration which is common to root and non-root loggers.
763
level = config.get('level', None)
764
if level is not None:
765
logger.setLevel(logging._checkLevel(level))
767
#Remove any existing handlers
768
for h in logger.handlers[:]:
769
logger.removeHandler(h)
770
handlers = config.get('handlers', None)
772
self.add_handlers(logger, handlers)
773
filters = config.get('filters', None)
775
self.add_filters(logger, filters)
777
def configure_logger(self, name, config, incremental=False):
778
"""Configure a non-root logger from a dictionary."""
779
logger = logging.getLogger(name)
780
self.common_logger_config(logger, config, incremental)
781
propagate = config.get('propagate', None)
782
if propagate is not None:
783
logger.propagate = propagate
785
def configure_root(self, config, incremental=False):
786
"""Configure a root logger from a dictionary."""
787
root = logging.getLogger()
788
self.common_logger_config(root, config, incremental)
790
dictConfigClass = DictConfigurator
792
def dictConfig(config):
793
"""Configure logging using a dictionary."""
794
dictConfigClass(config).configure()
797
def listen(port=DEFAULT_LOGGING_CONFIG_PORT):
799
Start up a socket server on the specified port, and listen for new
802
These will be sent as a file suitable for processing by fileConfig().
803
Returns a Thread object on which you can call start() to start the server,
804
and which you can join() when appropriate. To stop the server, call
808
raise NotImplementedError("listen() needs threading to work")
810
class ConfigStreamHandler(StreamRequestHandler):
812
Handler for a logging configuration request.
814
It expects a completely new logging configuration and uses fileConfig
821
Each request is expected to be a 4-byte length, packed using
822
struct.pack(">L", n), followed by the config file.
823
Uses fileConfig() to do the grunt work.
827
conn = self.connection
830
slen = struct.unpack(">L", chunk)[0]
831
chunk = self.connection.recv(slen)
832
while len(chunk) < slen:
833
chunk = chunk + conn.recv(slen - len(chunk))
837
assert isinstance(d, dict)
840
#Apply new configuration.
842
file = cStringIO.StringIO(chunk)
845
except (KeyboardInterrupt, SystemExit):
848
traceback.print_exc()
849
if self.server.ready:
850
self.server.ready.set()
851
except socket.error as e:
852
if e.errno != RESET_ERROR:
855
class ConfigSocketReceiver(ThreadingTCPServer):
857
A simple TCP socket-based logging config receiver.
860
allow_reuse_address = 1
862
def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT,
863
handler=None, ready=None):
864
ThreadingTCPServer.__init__(self, (host, port), handler)
865
logging._acquireLock()
867
logging._releaseLock()
871
def serve_until_stopped(self):
875
rd, wr, ex = select.select([self.socket.fileno()],
879
self.handle_request()
880
logging._acquireLock()
882
logging._releaseLock()
885
class Server(threading.Thread):
887
def __init__(self, rcvr, hdlr, port):
888
super(Server, self).__init__()
892
self.ready = threading.Event()
895
server = self.rcvr(port=self.port, handler=self.hdlr,
898
self.port = server.server_address[1]
901
logging._acquireLock()
903
logging._releaseLock()
904
server.serve_until_stopped()
906
return Server(ConfigSocketReceiver, ConfigStreamHandler, port)
910
Stop the listening server which was created with a call to listen().
913
logging._acquireLock()
919
logging._releaseLock()
1
# Copyright 2001-2014 by Vinay Sajip. All Rights Reserved.
3
# Permission to use, copy, modify, and distribute this software and its
4
# documentation for any purpose and without fee is hereby granted,
5
# provided that the above copyright notice appear in all copies and that
6
# both that copyright notice and this permission notice appear in
7
# supporting documentation, and that the name of Vinay Sajip
8
# not be used in advertising or publicity pertaining to distribution
9
# of the software without specific, written prior permission.
10
# VINAY SAJIP DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
11
# ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
12
# VINAY SAJIP BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR
13
# ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
14
# IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
15
# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18
Configuration functions for the logging package for Python. The core package
19
is based on PEP 282 and comments thereto in comp.lang.python, and influenced
20
by Apache's log4j system.
22
Copyright (C) 2001-2014 Vinay Sajip. All Rights Reserved.
24
To use, simply 'import logging' and log away!
31
import logging.handlers
46
from SocketServer import ThreadingTCPServer, StreamRequestHandler
49
DEFAULT_LOGGING_CONFIG_PORT = 9030
51
RESET_ERROR = errno.ECONNRESET
54
# The following code implements a socket listener for on-the-fly
55
# reconfiguration of logging.
57
# _listener holds the server object doing the listening
60
def fileConfig(fname, defaults=None, disable_existing_loggers=True):
62
Read the logging configuration from a ConfigParser-format file.
64
This can be called several times from an application, allowing an end user
65
the ability to select from various pre-canned configurations (if the
66
developer provides a mechanism to present the choices and load the chosen
71
cp = ConfigParser.ConfigParser(defaults)
72
if hasattr(fname, 'readline'):
77
formatters = _create_formatters(cp)
80
logging._acquireLock()
82
logging._handlers.clear()
83
del logging._handlerList[:]
84
# Handlers add themselves to logging._handlers
85
handlers = _install_handlers(cp, formatters)
86
_install_loggers(cp, handlers, disable_existing_loggers)
88
logging._releaseLock()
92
"""Resolve a dotted name to a global object."""
93
name = name.split('.')
95
found = __import__(used)
99
found = getattr(found, n)
100
except AttributeError:
102
found = getattr(found, n)
105
def _strip_spaces(alist):
106
return map(lambda x: x.strip(), alist)
109
return s if isinstance(s, str) else s.encode('utf-8')
111
def _create_formatters(cp):
112
"""Create and return formatters"""
113
flist = cp.get("formatters", "keys")
116
flist = flist.split(",")
117
flist = _strip_spaces(flist)
120
sectname = "formatter_%s" % form
121
opts = cp.options(sectname)
123
fs = cp.get(sectname, "format", 1)
126
if "datefmt" in opts:
127
dfs = cp.get(sectname, "datefmt", 1)
130
c = logging.Formatter
132
class_name = cp.get(sectname, "class")
134
c = _resolve(class_name)
140
def _install_handlers(cp, formatters):
141
"""Install and return handlers"""
142
hlist = cp.get("handlers", "keys")
145
hlist = hlist.split(",")
146
hlist = _strip_spaces(hlist)
148
fixups = [] #for inter-handler references
150
sectname = "handler_%s" % hand
151
klass = cp.get(sectname, "class")
152
opts = cp.options(sectname)
153
if "formatter" in opts:
154
fmt = cp.get(sectname, "formatter")
158
klass = eval(klass, vars(logging))
159
except (AttributeError, NameError):
160
klass = _resolve(klass)
161
args = cp.get(sectname, "args")
162
args = eval(args, vars(logging))
165
level = cp.get(sectname, "level")
166
h.setLevel(logging._levelNames[level])
168
h.setFormatter(formatters[fmt])
169
if issubclass(klass, logging.handlers.MemoryHandler):
171
target = cp.get(sectname,"target")
174
if len(target): #the target handler may not be loaded yet, so keep for later...
175
fixups.append((h, target))
177
#now all handlers are loaded, fixup inter-handler references...
179
h.setTarget(handlers[t])
183
def _install_loggers(cp, handlers, disable_existing_loggers):
184
"""Create and install loggers"""
186
# configure the root first
187
llist = cp.get("loggers", "keys")
188
llist = llist.split(",")
189
llist = list(map(lambda x: x.strip(), llist))
191
sectname = "logger_root"
194
opts = cp.options(sectname)
196
level = cp.get(sectname, "level")
197
log.setLevel(logging._levelNames[level])
198
for h in root.handlers[:]:
199
root.removeHandler(h)
200
hlist = cp.get(sectname, "handlers")
202
hlist = hlist.split(",")
203
hlist = _strip_spaces(hlist)
205
log.addHandler(handlers[hand])
207
#and now the others...
208
#we don't want to lose the existing loggers,
209
#since other threads may have pointers to them.
210
#existing is set to contain all existing loggers,
211
#and as we go through the new configuration we
212
#remove any which are configured. At the end,
213
#what's left in existing is the set of loggers
214
#which were in the previous configuration but
215
#which are not in the new configuration.
216
existing = list(root.manager.loggerDict.keys())
217
#The list needs to be sorted so that we can
218
#avoid disabling child loggers of explicitly
219
#named loggers. With a sorted list it is easier
220
#to find the child loggers.
222
#We'll keep the list of existing loggers
223
#which are children of named loggers here...
225
#now set up the new ones...
227
sectname = "logger_%s" % log
228
qn = cp.get(sectname, "qualname")
229
opts = cp.options(sectname)
230
if "propagate" in opts:
231
propagate = cp.getint(sectname, "propagate")
234
logger = logging.getLogger(qn)
236
i = existing.index(qn) + 1 # start with the entry after qn
238
pflen = len(prefixed)
239
num_existing = len(existing)
240
while i < num_existing:
241
if existing[i][:pflen] == prefixed:
242
child_loggers.append(existing[i])
246
level = cp.get(sectname, "level")
247
logger.setLevel(logging._levelNames[level])
248
for h in logger.handlers[:]:
249
logger.removeHandler(h)
250
logger.propagate = propagate
252
hlist = cp.get(sectname, "handlers")
254
hlist = hlist.split(",")
255
hlist = _strip_spaces(hlist)
257
logger.addHandler(handlers[hand])
259
#Disable any old loggers. There's no point deleting
260
#them as other threads may continue to hold references
261
#and by disabling them, you stop them doing any logging.
262
#However, don't disable children of named loggers, as that's
263
#probably not what was intended by the user.
265
logger = root.manager.loggerDict[log]
266
if log in child_loggers:
267
logger.level = logging.NOTSET
271
logger.disabled = disable_existing_loggers
275
IDENTIFIER = re.compile('^[a-z_][a-z0-9_]*$', re.I)
279
m = IDENTIFIER.match(s)
281
raise ValueError('Not a valid Python identifier: %r' % s)
285
class ConvertingMixin(object):
286
"""For ConvertingXXX's, this mixin class provides common functions"""
288
def convert_with_key(self, key, value, replace=True):
289
result = self.configurator.convert(value)
290
#If the converted value is different, save for next time
291
if value is not result:
294
if type(result) in (ConvertingDict, ConvertingList,
300
def convert(self, value):
301
result = self.configurator.convert(value)
302
if value is not result:
303
if type(result) in (ConvertingDict, ConvertingList,
309
# The ConvertingXXX classes are wrappers around standard Python containers,
310
# and they serve to convert any suitable values in the container. The
311
# conversion converts base dicts, lists and tuples to their wrapped
312
# equivalents, whereas strings which match a conversion format are converted
315
# Each wrapper should have a configurator attribute holding the actual
316
# configurator to use for conversion.
318
class ConvertingDict(dict, ConvertingMixin):
319
"""A converting dictionary wrapper."""
321
def __getitem__(self, key):
322
value = dict.__getitem__(self, key)
323
return self.convert_with_key(key, value)
325
def get(self, key, default=None):
326
value = dict.get(self, key, default)
327
return self.convert_with_key(key, value)
329
def pop(self, key, default=None):
330
value = dict.pop(self, key, default)
331
return self.convert_with_key(key, value, replace=False)
333
class ConvertingList(list, ConvertingMixin):
334
"""A converting list wrapper."""
335
def __getitem__(self, key):
336
value = list.__getitem__(self, key)
337
return self.convert_with_key(key, value)
339
def pop(self, idx=-1):
340
value = list.pop(self, idx)
341
return self.convert(value)
343
class ConvertingTuple(tuple, ConvertingMixin):
344
"""A converting tuple wrapper."""
345
def __getitem__(self, key):
346
value = tuple.__getitem__(self, key)
347
# Can't replace a tuple entry.
348
return self.convert_with_key(key, value, replace=False)
350
class BaseConfigurator(object):
352
The configurator base class which defines some useful defaults.
355
CONVERT_PATTERN = re.compile(r'^(?P<prefix>[a-z]+)://(?P<suffix>.*)$')
357
WORD_PATTERN = re.compile(r'^\s*(\w+)\s*')
358
DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*')
359
INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*')
360
DIGIT_PATTERN = re.compile(r'^\d+$')
363
'ext' : 'ext_convert',
364
'cfg' : 'cfg_convert',
367
# We might want to use a different one, e.g. importlib
368
importer = __import__
370
def __init__(self, config):
371
self.config = ConvertingDict(config)
372
self.config.configurator = self
373
# Issue 12718: winpdb replaces __import__ with a Python function, which
374
# ends up being treated as a bound method. To avoid problems, we
375
# set the importer on the instance, but leave it defined in the class
376
# so existing code doesn't break
377
if type(__import__) == types.FunctionType:
378
self.importer = __import__
380
def resolve(self, s):
382
Resolve strings to objects using standard import and attribute
388
found = self.importer(used)
392
found = getattr(found, frag)
393
except AttributeError:
395
found = getattr(found, frag)
398
e, tb = sys.exc_info()[1:]
399
v = ValueError('Cannot resolve %r: %s' % (s, e))
400
v.__cause__, v.__traceback__ = e, tb
403
def ext_convert(self, value):
404
"""Default converter for the ext:// protocol."""
405
return self.resolve(value)
407
def cfg_convert(self, value):
408
"""Default converter for the cfg:// protocol."""
410
m = self.WORD_PATTERN.match(rest)
412
raise ValueError("Unable to convert %r" % value)
414
rest = rest[m.end():]
415
d = self.config[m.groups()[0]]
418
m = self.DOT_PATTERN.match(rest)
422
m = self.INDEX_PATTERN.match(rest)
425
if not self.DIGIT_PATTERN.match(idx):
429
n = int(idx) # try as number first (most likely)
434
rest = rest[m.end():]
436
raise ValueError('Unable to convert '
437
'%r at %r' % (value, rest))
438
#rest should be empty
441
def convert(self, value):
443
Convert values to an appropriate type. dicts, lists and tuples are
444
replaced by their converting alternatives. Strings are checked to
445
see if they have a conversion format and are converted if they do.
447
if not isinstance(value, ConvertingDict) and isinstance(value, dict):
448
value = ConvertingDict(value)
449
value.configurator = self
450
elif not isinstance(value, ConvertingList) and isinstance(value, list):
451
value = ConvertingList(value)
452
value.configurator = self
453
elif not isinstance(value, ConvertingTuple) and\
454
isinstance(value, tuple):
455
value = ConvertingTuple(value)
456
value.configurator = self
457
elif isinstance(value, basestring): # str for py3k
458
m = self.CONVERT_PATTERN.match(value)
462
converter = self.value_converters.get(prefix, None)
465
converter = getattr(self, converter)
466
value = converter(suffix)
469
def configure_custom(self, config):
470
"""Configure an object with a user-supplied factory."""
472
if not hasattr(c, '__call__') and hasattr(types, 'ClassType') and type(c) != types.ClassType:
474
props = config.pop('.', None)
475
# Check for valid identifiers
476
kwargs = dict([(k, config[k]) for k in config if valid_ident(k)])
479
for name, value in props.items():
480
setattr(result, name, value)
483
def as_tuple(self, value):
484
"""Utility function which converts lists to tuples."""
485
if isinstance(value, list):
489
class DictConfigurator(BaseConfigurator):
491
Configure logging using a dictionary-like object to describe the
496
"""Do the configuration."""
499
if 'version' not in config:
500
raise ValueError("dictionary doesn't specify a version")
501
if config['version'] != 1:
502
raise ValueError("Unsupported version: %s" % config['version'])
503
incremental = config.pop('incremental', False)
505
logging._acquireLock()
508
handlers = config.get('handlers', EMPTY_DICT)
509
for name in handlers:
510
if name not in logging._handlers:
511
raise ValueError('No handler found with '
515
handler = logging._handlers[name]
516
handler_config = handlers[name]
517
level = handler_config.get('level', None)
519
handler.setLevel(logging._checkLevel(level))
520
except StandardError as e:
521
raise ValueError('Unable to configure handler '
522
'%r: %s' % (name, e))
523
loggers = config.get('loggers', EMPTY_DICT)
526
self.configure_logger(name, loggers[name], True)
527
except StandardError as e:
528
raise ValueError('Unable to configure logger '
529
'%r: %s' % (name, e))
530
root = config.get('root', None)
533
self.configure_root(root, True)
534
except StandardError as e:
535
raise ValueError('Unable to configure root '
538
disable_existing = config.pop('disable_existing_loggers', True)
540
logging._handlers.clear()
541
del logging._handlerList[:]
543
# Do formatters first - they don't refer to anything else
544
formatters = config.get('formatters', EMPTY_DICT)
545
for name in formatters:
547
formatters[name] = self.configure_formatter(
549
except StandardError as e:
550
raise ValueError('Unable to configure '
551
'formatter %r: %s' % (name, e))
552
# Next, do filters - they don't refer to anything else, either
553
filters = config.get('filters', EMPTY_DICT)
556
filters[name] = self.configure_filter(filters[name])
557
except StandardError as e:
558
raise ValueError('Unable to configure '
559
'filter %r: %s' % (name, e))
561
# Next, do handlers - they refer to formatters and filters
562
# As handlers can refer to other handlers, sort the keys
563
# to allow a deterministic order of configuration
564
handlers = config.get('handlers', EMPTY_DICT)
566
for name in sorted(handlers):
568
handler = self.configure_handler(handlers[name])
570
handlers[name] = handler
571
except StandardError as e:
572
if 'target not configured yet' in str(e):
573
deferred.append(name)
575
raise ValueError('Unable to configure handler '
576
'%r: %s' % (name, e))
578
# Now do any that were deferred
579
for name in deferred:
581
handler = self.configure_handler(handlers[name])
583
handlers[name] = handler
584
except StandardError as e:
585
raise ValueError('Unable to configure handler '
586
'%r: %s' % (name, e))
588
# Next, do loggers - they refer to handlers and filters
590
#we don't want to lose the existing loggers,
591
#since other threads may have pointers to them.
592
#existing is set to contain all existing loggers,
593
#and as we go through the new configuration we
594
#remove any which are configured. At the end,
595
#what's left in existing is the set of loggers
596
#which were in the previous configuration but
597
#which are not in the new configuration.
599
existing = root.manager.loggerDict.keys()
600
#The list needs to be sorted so that we can
601
#avoid disabling child loggers of explicitly
602
#named loggers. With a sorted list it is easier
603
#to find the child loggers.
605
#We'll keep the list of existing loggers
606
#which are children of named loggers here...
608
#now set up the new ones...
609
loggers = config.get('loggers', EMPTY_DICT)
611
name = _encoded(name)
613
i = existing.index(name)
614
prefixed = name + "."
615
pflen = len(prefixed)
616
num_existing = len(existing)
617
i = i + 1 # look at the entry after name
618
while (i < num_existing) and\
619
(existing[i][:pflen] == prefixed):
620
child_loggers.append(existing[i])
622
existing.remove(name)
624
self.configure_logger(name, loggers[name])
625
except StandardError as e:
626
raise ValueError('Unable to configure logger '
627
'%r: %s' % (name, e))
629
#Disable any old loggers. There's no point deleting
630
#them as other threads may continue to hold references
631
#and by disabling them, you stop them doing any logging.
632
#However, don't disable children of named loggers, as that's
633
#probably not what was intended by the user.
635
logger = root.manager.loggerDict[log]
636
if log in child_loggers:
637
logger.level = logging.NOTSET
639
logger.propagate = True
640
elif disable_existing:
641
logger.disabled = True
643
# And finally, do the root logger
644
root = config.get('root', None)
647
self.configure_root(root)
648
except StandardError as e:
649
raise ValueError('Unable to configure root '
652
logging._releaseLock()
654
def configure_formatter(self, config):
655
"""Configure a formatter from a dictionary."""
657
factory = config['()'] # for use in exception handler
659
result = self.configure_custom(config)
660
except TypeError as te:
661
if "'format'" not in str(te):
663
#Name of parameter changed from fmt to format.
664
#Retry with old name.
665
#This is so that code can be used with older Python versions
667
config['fmt'] = config.pop('format')
668
config['()'] = factory
669
result = self.configure_custom(config)
671
fmt = config.get('format', None)
672
dfmt = config.get('datefmt', None)
673
result = logging.Formatter(fmt, dfmt)
676
def configure_filter(self, config):
677
"""Configure a filter from a dictionary."""
679
result = self.configure_custom(config)
681
name = config.get('name', '')
682
result = logging.Filter(name)
685
def add_filters(self, filterer, filters):
686
"""Add filters to a filterer from a list of names."""
689
filterer.addFilter(self.config['filters'][f])
690
except StandardError as e:
691
raise ValueError('Unable to add filter %r: %s' % (f, e))
693
def configure_handler(self, config):
694
"""Configure a handler from a dictionary."""
695
formatter = config.pop('formatter', None)
698
formatter = self.config['formatters'][formatter]
699
except StandardError as e:
700
raise ValueError('Unable to set formatter '
701
'%r: %s' % (formatter, e))
702
level = config.pop('level', None)
703
filters = config.pop('filters', None)
706
if not hasattr(c, '__call__') and hasattr(types, 'ClassType') and type(c) != types.ClassType:
710
cname = config.pop('class')
711
klass = self.resolve(cname)
712
#Special case for handler which refers to another handler
713
if issubclass(klass, logging.handlers.MemoryHandler) and\
716
th = self.config['handlers'][config['target']]
717
if not isinstance(th, logging.Handler):
718
config['class'] = cname # restore for deferred configuration
719
raise StandardError('target not configured yet')
720
config['target'] = th
721
except StandardError as e:
722
raise ValueError('Unable to set target handler '
723
'%r: %s' % (config['target'], e))
724
elif issubclass(klass, logging.handlers.SMTPHandler) and\
725
'mailhost' in config:
726
config['mailhost'] = self.as_tuple(config['mailhost'])
727
elif issubclass(klass, logging.handlers.SysLogHandler) and\
729
config['address'] = self.as_tuple(config['address'])
731
kwargs = dict([(k, config[k]) for k in config if valid_ident(k)])
733
result = factory(**kwargs)
734
except TypeError as te:
735
if "'stream'" not in str(te):
737
#The argument name changed from strm to stream
738
#Retry with old name.
739
#This is so that code can be used with older Python versions
741
kwargs['strm'] = kwargs.pop('stream')
742
result = factory(**kwargs)
744
result.setFormatter(formatter)
745
if level is not None:
746
result.setLevel(logging._checkLevel(level))
748
self.add_filters(result, filters)
751
def add_handlers(self, logger, handlers):
752
"""Add handlers to a logger from a list of names."""
755
logger.addHandler(self.config['handlers'][h])
756
except StandardError as e:
757
raise ValueError('Unable to add handler %r: %s' % (h, e))
759
def common_logger_config(self, logger, config, incremental=False):
761
Perform configuration which is common to root and non-root loggers.
763
level = config.get('level', None)
764
if level is not None:
765
logger.setLevel(logging._checkLevel(level))
767
#Remove any existing handlers
768
for h in logger.handlers[:]:
769
logger.removeHandler(h)
770
handlers = config.get('handlers', None)
772
self.add_handlers(logger, handlers)
773
filters = config.get('filters', None)
775
self.add_filters(logger, filters)
777
def configure_logger(self, name, config, incremental=False):
778
"""Configure a non-root logger from a dictionary."""
779
logger = logging.getLogger(name)
780
self.common_logger_config(logger, config, incremental)
781
propagate = config.get('propagate', None)
782
if propagate is not None:
783
logger.propagate = propagate
785
def configure_root(self, config, incremental=False):
786
"""Configure a root logger from a dictionary."""
787
root = logging.getLogger()
788
self.common_logger_config(root, config, incremental)
790
dictConfigClass = DictConfigurator
792
def dictConfig(config):
793
"""Configure logging using a dictionary."""
794
dictConfigClass(config).configure()
797
def listen(port=DEFAULT_LOGGING_CONFIG_PORT):
799
Start up a socket server on the specified port, and listen for new
802
These will be sent as a file suitable for processing by fileConfig().
803
Returns a Thread object on which you can call start() to start the server,
804
and which you can join() when appropriate. To stop the server, call
808
raise NotImplementedError("listen() needs threading to work")
810
class ConfigStreamHandler(StreamRequestHandler):
812
Handler for a logging configuration request.
814
It expects a completely new logging configuration and uses fileConfig
821
Each request is expected to be a 4-byte length, packed using
822
struct.pack(">L", n), followed by the config file.
823
Uses fileConfig() to do the grunt work.
827
conn = self.connection
830
slen = struct.unpack(">L", chunk)[0]
831
chunk = self.connection.recv(slen)
832
while len(chunk) < slen:
833
chunk = chunk + conn.recv(slen - len(chunk))
837
assert isinstance(d, dict)
840
#Apply new configuration.
842
file = cStringIO.StringIO(chunk)
845
except (KeyboardInterrupt, SystemExit):
848
traceback.print_exc()
849
if self.server.ready:
850
self.server.ready.set()
851
except socket.error as e:
852
if e.errno != RESET_ERROR:
855
class ConfigSocketReceiver(ThreadingTCPServer):
857
A simple TCP socket-based logging config receiver.
860
allow_reuse_address = 1
862
def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT,
863
handler=None, ready=None):
864
ThreadingTCPServer.__init__(self, (host, port), handler)
865
logging._acquireLock()
867
logging._releaseLock()
871
def serve_until_stopped(self):
875
rd, wr, ex = select.select([self.socket.fileno()],
879
self.handle_request()
880
logging._acquireLock()
882
logging._releaseLock()
885
class Server(threading.Thread):
887
def __init__(self, rcvr, hdlr, port):
888
super(Server, self).__init__()
892
self.ready = threading.Event()
895
server = self.rcvr(port=self.port, handler=self.hdlr,
898
self.port = server.server_address[1]
901
logging._acquireLock()
903
logging._releaseLock()
904
server.serve_until_stopped()
906
return Server(ConfigSocketReceiver, ConfigStreamHandler, port)
910
Stop the listening server which was created with a call to listen().
913
logging._acquireLock()
919
logging._releaseLock()
added
removed
python/Lib/logging/config.py
