Lazy Loading PyQt Data Models (for QTreeViews)

Sometimes loading all of the required data into a model at the time it’s created is not a great option.  For example, you wouldn’t want to have a file system model enumerating every file – this could take quite a while, and besides, the user is not likely to want to navigate to every file on the filesystem!  A better option is to lazy load the data as required.

Qt data models allow you to do this by reimplementing the virtual functions rowCount , hasChildren , canFetchMore  and fetchMore  of QAbstractItemModel

  • rowCount – you would normally implement this for a tree model anyway. Just make sure that it returns 0 if the node has children that have not been loaded yet.
  • hasChildren – override to return True for nodes that have children that haven’t been loaded yet and return whatever the base class returns in all other cases.
  • canFetchMore – return True if the node has children that haven’t been loaded yet, False otherwise.
  • fetchMore – this is where you perform whatever logic you need to decide what nodes to create and insert them into the model.

Here’s the basic idea – for nodes that you know have children that haven’t been loaded, return 0 from rowCount and True from canFetchMore and hasChildren. This tells Qt to show a node with an expander next to it even though it currently has no children. When the expander is clicked, fetchMore is called and you populate the children from the given parent.

One thing to note – you must call beginInsertRows and endInsertRows in the fetchMore method. What’s more, you musn’t change the underlying datastore before calling beginInsertRows or after endInsertRows. Unfortunately, you need to know how many rows you are inserting when you call beginInsertRows – so you are probably going to want to generate a list of nodes to add, then make the call to beginInsertRows. If you do it this way though, you can’t set the new nodes’ parent, as it would change the underlying datastore.

The item model is really the only thing that changes from a normal tree model with a couple of state attributes in the node for tracking whether data has been loaded or not.  Code below with comments on the model.

 

import sys

from PyQt4.QtGui import QApplication, QTreeView

from node import Node
from model import FileSystemTreeModel


app = QApplication(sys.argv)

model = FileSystemTreeModel(Node('Filename'), path='c:/')


tree = QTreeView()
tree.setModel(model)

tree.show()

sys.exit(app.exec_())
import os

from PyQt4.QtCore import Qt, QModelIndex, QAbstractItemModel

from node import Node


class FileSystemTreeModel(QAbstractItemModel):

        FLAG_DEFAULT = Qt.ItemIsEnabled | Qt.ItemIsSelectable

        def __init__(self, root, path='c:/', parent=None):
            super(FileSystemTreeModel, self).__init__()

            self.root = root
            self.parent = parent
            self.path = path

            # generate root node children
            for file in os.listdir(path):
                file_path = os.path.join(path, file)

                node = Node(file, file_path, parent=self.root)
                if os.path.isdir(file_path):
                    node.is_dir = True

        # takes a model index and returns the related Python node
        def getNode(self, index):
            if index.isValid():
                return index.internalPointer()
            else:
                return self.root

        # check if the note has data that has not been loaded yet
        def canFetchMore(self, index):
            node = self.getNode(index)

            if node.is_dir and not node.is_traversed:
                return True

            return False

        # called if canFetchMore returns True, then dynamically inserts nodes required for
        # directory contents
        def fetchMore(self, index):
            parent = self.getNode(index)

            nodes = []
            for file in os.listdir(parent.path):
                file_path = os.path.join(parent.path, file)

                node = Node(file, file_path)
                if os.path.isdir(file_path):
                    node.is_dir = True

                nodes.append(node)

            self.insertNodes(0, nodes, index)
            parent.is_traversed = True

        # returns True for directory nodes so that Qt knows to check if there is more to load
        def hasChildren(self, index):
            node = self.getNode(index)

            if node.is_dir:
                return True

            return super(FileSystemTreeModel, self).hasChildren(index)

        # should return 0 if there is data to fetch (handled implicitly by check length of child list)
        def rowCount(self, parent):
            node = self.getNode(parent)
            return node.child_count()

        def columnCount(self, parent):
            return 1

        def flags(self, index):
            return FileSystemTreeModel.FLAG_DEFAULT

        def parent(self, index):
            node = self.getNode(index)

            parent = node.parent
            if parent == self.root:
                return QModelIndex()

            return self.createIndex(parent.row(), 0, parent)

        def index(self, row, column, parent):
            node = self.getNode(parent)

            child = node.child(row)

            if not child:
                return QModelIndex()

            return self.createIndex(row, column, child)

        def headerData(self, section, orientation, role):
            return self.root.name

        def data(self, index, role):
            if not index.isValid():
                return None

            node = index.internalPointer()

            if role == Qt.DisplayRole:
                return node.name

            else:
                return None

        def insertNodes(self, position, nodes, parent=QModelIndex()):
            node = self.getNode(parent)

            self.beginInsertRows(parent, position, position + len(nodes) - 1)

            for child in nodes:
                success = node.insert_child(position, child)

            self.endInsertRows()

            return True
class Node(object):

        def __init__(self, name, path=None, parent=None):
            super(Node, self).__init__()

            self.name = name
            self.children = []
            self.parent = parent

            self.is_dir = False
            self.path = path
            self.is_traversed = False

            if parent is not None:
                parent.add_child(self)

        def add_child(self, child):
            self.children.append(child)
            child.parent = self

        def insert_child(self, position, child):
            if position < 0 or position > self.child_count():
                return False

            self.children.insert(position, child)
            child.parent = self

            return True

        def child(self, row):
            return self.children
def child_count(self): return len(self.children) def row(self): if self.parent is not None: return self.parent.children.index(self) return 0

 

Lazy Loading PyQt Data Models (for QTreeViews)

Leave a Reply