%load mod1.py

class SpecFile(object):
    class Section(object):
        def __init__(self): pass
        def validate(self): pass
        
    class DynamicSection(Section):
        def __init__(self): pass
        
    class StaticSection(Section):
        def __init__(self): pass
        
    class Section1(StaticSection):
        def __init__(self): pass
        def validate(self): pass
        
    class Section2(DynamicSection):
        def __init__(self): pass
        def validate(self): pass
        
def main():
    pass


%load nbcommon.py

# custom exception class for no docstring
class NoDocstrError(Exception): pass

def has_docstr(entity):
    """ Check whether this entity has a docstring """
    docstr = entity.__doc__
    return docstr != None and docstr.strip() != ''

import inspect

def get_entity_type(entity):
    """ Check whether entity is supported for docstring heck """
    for entity_type in ['module', 'function', 'class', 'method'] :
        # inspect module has inspect.ismodule, inspect.isfunction - leverage that
        inspect_func = getattr(inspect, 'is' + entity_type)
        if inspect_func(entity): return entity_type
    raise ValueError('Invalid entity: %s passed' % entity)


%load t1.py

from nbcommon import *
import mod1

import inspect
from collections import defaultdict

class Modstruct1(object):
    """ Return a data structure representing all members of the passed
    entity """

    def __init__(self, base_entity):
        self.base_entity = base_entity

    def get_all_members(self):
        """ Get all the members (nested also) of the passed entity """
        return inspect.getmembers(self.base_entity)


class TestDocstr1(object):

    @classmethod
    def test_docstr(self, entity):
        """ Test whether the passed in entity and its children have docstring """
        entity_type = None
        non_docstr_entities = defaultdict(list)
        all_members = Modstruct1(entity).get_all_members()

        # get all the members of the passed entity
        for member in all_members:
            ref = member[1]
            try:
                entity_type = get_entity_type(ref)
                if not has_docstr(ref):
                    non_docstr_entities[entity_type].append(ref)
            except ValueError:
                # invalid entity type - skip it
                continue

        # if any entities without docstring - consolidate and raise error
        if non_docstr_entities.keys():
            errors = []
            for entity_type, refs in non_docstr_entities.iteritems():
                for ref in refs:
                    errors.append('%s %s does not have docstr' % (entity_type,
                                                                  ref.__name__))
            raise NoDocstrError('\n'.join(errors))

        return True

TestDocstr1.test_docstr(mod1)


%load t2.py

from nbcommon import *
import mod1

import inspect
import sys
from collections import defaultdict

class Modstruct2(object):
    """ Return a data structure representing all members of the passed
    entity """

    def __init__(self, base_entity):
        self.base_entity_type = get_entity_type(base_entity)
        self.base_entity = base_entity
        self.base_module = base_entity
        if self.base_entity_type != 'module':
            # if entity_type is class - know which module it belongs to
            self.base_module = sys.modules[base_entity.__module__]

    def get_entity_members(self, entity):
        """ Get first level members of the passed entity """
        members = []
        parent_name = entity.__name__
        for member in inspect.getmembers(entity):
            ref = member[1]
            # member has to be of supported entity type
            try:
                ref_type = get_entity_type(ref)
            except ValueError:
                continue

            # we will not inspect modules imported in base module
            if inspect.ismodule(ref): continue

            # member has to be defined in base module
            if ref.__module__ != self.base_module.__name__: continue

            # valid member - construct member data
            member_data = {
                'type': ref_type, 
                'ref': ref, 
                'name': entity.__name__ + '.' + ref.__name__,
                'parent_ref': entity,
                'parent_name': parent_name,
            }
            members.append(member_data)
        return members

    def get_all_members(self):
        """ Get all the members (nested also) of the passed entity """
        # add base module as the first element
        all_members = [{'type': 'module',
                        'ref': self.base_module, 
                        'name': self.base_module.__name__,
                        'parent_ref': None,
                        'parent_name': None}]

        # get first level members of the main entity
        nested_members = self.get_entity_members(self.base_entity)
        all_members.extend(nested_members)

        # call get_entity_members repetitively till you reach a stage where 
        # there are no nested members
        while nested_members:
            curr_nested_members = []
            for member_data in nested_members:
                if member_data['type'] == 'class':
                    # drill nested members only in a class
                    members = self.get_entity_members(member_data['ref'])
                    curr_nested_members.extend(members)
            nested_members = curr_nested_members
            all_members.extend(nested_members)

        return all_members

class TestDocstr2(object):

    @classmethod
    def test_docstr(self, entity):
        all_members = Modstruct2(entity).get_all_members()

        non_docstr_entities = defaultdict(list)

        # get all the nested members of root entity
        for member_data in all_members:
            # consolidate members based on type
            if not has_docstr(member_data['ref']):
                member_name = member_data['name']
                non_docstr_entities[member_data['type']].append(member_name)

        if non_docstr_entities.keys():
            errors = []
            # create error string
            for entity_type, refs in non_docstr_entities.iteritems():
                for refname in refs:
                    errors.append('%s: %s does not have docstr' % (entity_type,
                                                                   refname))
            raise NoDocstrError('\n' + '\n'.join(errors))
        return True

TestDocstr2.test_docstr(mod1)


%load t3.py

from nbcommon import *
import mod1

import inspect
import sys
from collections import defaultdict

class Modstruct3(object):
    """ Return a data structure representing all members of the passed
    entity """

    def __init__(self, base_entity):
        self.base_entity_type = get_entity_type(base_entity)
        self.base_entity = base_entity
        self.base_module = base_entity
        self.id_name_map = {}
        self.all_members = []
        if self.base_entity_type != 'module':
            # if entity_type is class - know which module it belongs to
            self.base_module = sys.modules[base_entity.__module__]

    def get_entity_name(self, entity):
        """ Return fully qualified name of entity """
        return self.id_name_map.get(id(entity), None)

    def build_id_name_map(self, entity, parent=None):
        """ Map entity id to its fully qualified name """
        entity_name = entity.__name__
        if not parent is None:
            id_parent = id(parent)
            if id_parent in self.id_name_map:
                parent_name = self.id_name_map[id_parent]
                entity_name = '.'.join([parent_name, entity.__name__])
        self.id_name_map[id(entity)] = entity_name

    def extract_entity_members(self):
        """ From all the members extract out member tree of the base 
        entity """
        if self.base_entity_type == 'module':
            self.base_entity_members = self.all_members
            return self.base_entity_members

        base_entity_name = self.get_entity_name(self.base_entity)

        base_entity_members = []
        for member in self.all_members:
            if member['name'].startswith(base_entity_name):
                base_entity_members.append(member)
        self.base_entity_members = base_entity_members

    def get_entity_members(self, entity):
        """ Get first level members of the passed entity """
        members = []
        parent_name = self.get_entity_name(entity)
        for member in inspect.getmembers(entity):
            ref = member[1]
            # member has to be of supported entity type
            try:
                ref_type = get_entity_type(ref)
            except ValueError:
                continue

            # we will not inspect modules imported in base module
            if inspect.ismodule(ref): continue

            # member has to be defined in base module
            if ref.__module__ != self.base_module.__name__: continue

            # valid member - construct member data
            member_data = {
                'type': ref_type, 
                'ref': ref, 
                'name': parent_name + '.' + ref.__name__,
                'parent_ref': entity,
                'parent_name': parent_name
            }
            members.append(member_data)
            self.build_id_name_map(ref, entity)
        return members

    def get_all_members(self):
        """ Get all the members (nested also) of the passed entity """

        # add base module as the first element
        all_members = [{'type': 'module',
                        'ref': self.base_module, 
                        'name': self.base_module.__name__,
                        'parent_ref': None,
                        'parent_name': None}]

        # add base module as first entry to id_name_map - root of all names
        self.build_id_name_map(self.base_module, None)

        # get first level members of the module
        nested_members = self.get_entity_members(self.base_module)
        all_members.extend(nested_members)

        # call get_entity_members repetitively till you reach a stage where 
        # there are no nested members
        while nested_members:
            curr_nested_members = []
            # for member_type, member_ref, member_name in nested_members:
            for member_data in nested_members:
                if member_data['type'] == 'class':
                    # drill nested members only in a class
                    members = self.get_entity_members(member_data['ref'])
                    curr_nested_members.extend(members)
            nested_members = curr_nested_members
            all_members.extend(nested_members)

        self.all_members = all_members

        # extract subset of members in case base_entity is not a module
        self.extract_entity_members()

        return self.base_entity_members

class TestDocstr3(object):

    @classmethod
    def test_docstr(self, entity):
        all_members = Modstruct3(entity).get_all_members()

        non_docstr_entities = defaultdict(list)

        # get all the nested members of root entity
        for member_data in all_members:
            # consolidate members based on type
            if not has_docstr(member_data['ref']):
                member_name = member_data['name']
                non_docstr_entities[member_data['type']].append(member_name)

        if non_docstr_entities.keys():
            errors = []
            # create error string
            for entity_type, refs in non_docstr_entities.iteritems():
                for refname in refs:
                    errors.append('%s: %s does not have docstr' % (entity_type,
                                                                   refname))
            raise NoDocstrError('\n' + '\n'.join(errors))
        return True

TestDocstr3.test_docstr(mod1)


TestDocstr3.test_docstr(mod1.main) # Testing if the function has docstring

TestDocstr3.test_docstr(mod1.SpecFile) # Testing if the class and its members have docstring

TestDocstr3.test_docstr(mod1.SpecFile.Section.validate)

m1 = Modstruct3(mod1)
members = m1.get_all_members()
target_keys = ['mod1', 'mod1.main', 'mod1.SpecFile.Section', 'mod1.SpecFile.Section.validate']
target_members = {}
for member in members:
    # map id => name for target_keys members
    if member['name'] in target_keys:
        target_members[member['name']] = id(member['ref'])

id(mod1), target_members['mod1'] # same

id(mod1.main), target_members['mod1.main'] # same

id(mod1.SpecFile.Section), target_members['mod1.SpecFile.Section'] # same

id(mod1.SpecFile.Section.validate), target_members['mod1.SpecFile.Section.validate'] # diff

%load test_meth_id.py

def f1(): pass

class A():
    def m1(): pass

x = f1
y = f1
z = f1
print "\n==== id for function f1 ===="
print 'id(f1) = ' + str(id(f1))
print 'id(x) = ' + str(id(x))
print 'id(y) = ' + str(id(y))
print 'id(z) = ' + str(id(z))

x = A
y = A
z = A
print "\n==== id for class A ===="
print 'id(A) = ' + str(id(A))
print 'id(x) = ' + str(id(x))
print 'id(y) = ' + str(id(y))
print 'id(z) = ' + str(id(z))

x = A.m1
y = A.m1
z = A.m1
print "\n==== id for method A.m1 ===="
print 'id(A.m1) = ' + str(id(A.m1))
print 'id(x) = ' + str(id(x))
print 'id(y) = ' + str(id(y))
print 'id(z) = ' + str(id(z))

print 'x is y ' + str(x is y)
print 'x == y ' + str(x == y)


%load t4.py

from nbcommon import *
import mod1

import inspect
import sys
from collections import defaultdict

class Modstruct4(object):
    """ Return a data structure representing all members of the passed
    entity """

    def __init__(self, base_entity, **options):
        self.base_entity_type = get_entity_type(base_entity)
        self.base_entity = base_entity
        self.base_module = base_entity
        self.id_name_map = {}
        self.all_members = []
        self.options = {'categorize': False}
        self.options.update(options)
        if self.base_entity_type != 'module':
            # if entity_type is class - know which module it belongs to
            self.base_module = sys.modules[base_entity.__module__]

    def get_entity_name(self, entity):
        """ Return fully qualified name of entity """
        return self.id_name_map.get(id(entity), None)

    def get_base_entity_name(self):
        """ Return the name of the base entity passed in by the user """
        # if base entity is not a method - just look up its id
        if self.base_entity_type != 'method':
            return self.get_entity_name(self.base_entity)

        # else as method id does not stay constant, cycle through all members 
        # and return the member matching the base entity ref
        for member in self.all_members:
            if self.base_entity == member['ref']:
                return self.get_entity_name(member['ref'])

    def build_id_name_map(self, entity, parent=None):
        """ Map entity id to its fully qualified name """
        entity_name = entity.__name__
        if not parent is None:
            id_parent = id(parent)
            if id_parent in self.id_name_map:
                parent_name = self.id_name_map[id_parent]
                entity_name = '.'.join([parent_name, entity.__name__])
        self.id_name_map[id(entity)] = entity_name

    def extract_entity_members(self):
        """ From all the members extract out member tree of the base 
        entity """
        if self.base_entity_type == 'module':
            self.base_entity_members = self.all_members
            return self.base_entity_members

        base_entity_name = self.get_base_entity_name()

        base_entity_members = []
        for member in self.all_members:
            if member['name'].startswith(base_entity_name):
                base_entity_members.append(member)
        self.base_entity_members = base_entity_members

    def get_entity_members(self, entity):
        """ Get first level members of the passed entity """
        members = []
        parent_name = self.get_entity_name(entity)
        for member in inspect.getmembers(entity):
            ref = member[1]
            # member has to be of supported entity type
            try:
                ref_type = get_entity_type(ref)
            except ValueError:
                continue

            # we will not inspect modules imported in base module
            if inspect.ismodule(ref): continue

            # member has to be defined in base module
            if ref.__module__ != self.base_module.__name__: continue

            # valid member - construct member data
            member_data = {
                'type': ref_type, 
                'ref': ref, 
                'name': parent_name + '.' + ref.__name__,
                'parent_ref': entity,
                'parent_name': parent_name
            }
            members.append(member_data)
            self.build_id_name_map(ref, entity)
        return members

    def get_all_members(self):
        """ Get all the members (nested also) of the passed entity """

        # add base module as the first element
        all_members = [{'type': 'module',
                        'ref': self.base_module, 
                        'name': self.base_module.__name__,
                        'parent_ref': None,
                        'parent_name': None}]

        # add base module as first entry to id_name_map - root of all names
        self.build_id_name_map(self.base_module, None)

        # get first level members of the module
        nested_members = self.get_entity_members(self.base_module)
        all_members.extend(nested_members)

        # call get_entity_members repetitively till you reach a stage where 
        # there are no nested members
        while nested_members:
            curr_nested_members = []
            # for member_type, member_ref, member_name in nested_members:
            for member_data in nested_members:
                if member_data['type'] == 'class':
                    # drill nested members only in a class
                    members = self.get_entity_members(member_data['ref'])
                    curr_nested_members.extend(members)
            nested_members = curr_nested_members
            all_members.extend(nested_members)

        self.all_members = all_members

        # extract subset of members in case base_entity is not a module
        self.extract_entity_members()

        # categorize members if required
        if self.options['categorize']:
            return self.categorize()

        return self.base_entity_members

class TestDocstr4(object):

    @classmethod
    def test_docstr(self, entity):
        all_members = Modstruct4(entity).get_all_members()

        non_docstr_entities = defaultdict(list)

        # get all the nested members of root entity
        for member_data in all_members:
            # consolidate members based on type
            if not has_docstr(member_data['ref']):
                member_name = member_data['name']
                non_docstr_entities[member_data['type']].append(member_name)

        if non_docstr_entities.keys():
            errors = []
            # create error string
            for entity_type, refs in non_docstr_entities.iteritems():
                for refname in refs:
                    errors.append('%s: %s does not have docstr' % (entity_type,
                                                                   refname))
            raise NoDocstrError('\n' + '\n'.join(errors))
        return True

TestDocstr4.test_docstr(mod1.SpecFile.Section1.validate)